./0000755000175000017500000000000011463604477012005 5ustar silverdogsilverdog./sample.sh0000755000175000017500000000176311463604477013634 0ustar silverdogsilverdog#!/bin/sh cd MAILDIRSYNC_OPTS="-rvv -a md5 --backup=/tmp/emails_deleted --rsh-sep=, --rsh=ssh,-C" SLEEP_FOR=300 # MAILDIRSYNC_OPTS="-rvv" while true do # calculating the MD5 sum of the old status file MD5OLD=`gunzip laptop" maildirsync $MAILDIRSYNC_OPTS $@ desktop:Maildir Maildir lib/maildirsync_desktop_laptop.gz echo "`date`: laptop -> desktop" maildirsync $MAILDIRSYNC_OPTS $@ Maildir desktop:Maildir lib/maildirsync_laptop_desktop.gz # checking if the status file is changed. If it is changed, then we # restart the synchronization. (We synchronize until it is not changed) if [ "$MD5OLD" = "`gunzip &1 | tee -a "$HOME/maildirsync-`date +%Y%m%d`.$$.log" ./contrib/0000755000175000017500000000000011625002762013432 5ustar silverdogsilverdog./contrib/mailbalance-0.2.0/0000755000175000017500000000000011463604477016330 5ustar silverdogsilverdog./contrib/mailbalance-0.2.0/mailbalance0000755000175000017500000000151211463604477020505 0ustar silverdogsilverdog#!/usr/local/bin/bash ME=`whoami` if [ $ME != "root" ] ; then echo "You must be root to run this script" exit fi source ./mailbalance.conf source ./functions.inc if [ -z $1 ] ; then echo 'Usage: '$0' [-d] < `cat userlist.txt`' exit fi #Do not functionize this .. needs to be in global scope to shift off switches if [ "${1}" = "-d" ] ; then shift MODE=daemon; else MODE=verbose; fi echo "Starting $0 in ${MODE} mode..." while true ; do for i in $@ ; do DOSYNC=`checkuser $i`; if [ "${DOSYNC}" = "OK" ] ; then case $MODE in daemon) doit $i ${MODE} 2>&1 > /dev/null ;; *) doit $i ${MODE} ;; esac else writenok $i luserprep $i ruserprep $i #uncomment below for debugging, if you want to stop loop #exit writeok $i fi done done ./contrib/mailbalance-0.2.0/install.sh0000755000175000017500000000550111463604477020336 0ustar silverdogsilverdog#!/usr/local/bin/bash ME=`whoami` if [ $ME != "root" ] ; then echo "You must be root to run this script" exit fi echo "Have you configured mailbalance.conf? (yes or no)" read -e A if [ "${A}" = yes ] ; then echo "Proceeding..." else echo "You must configure all values in mailbalance.conf before running this..." exit fi if (whereis drsync.pl | egrep "drsync.pl$") 2>&1 > /dev/null ; then echo "drsync.pl found in runpath .. Continuing to install" ; else echo "Please install drsync-0.4.2 or greater before continuing ( http://hacks.dlux.hu/drsync/ ). Be sure that drsync.pl is in your runpath" ; echo "Note: You must install this package on both this slave and the master that it will sync with!!!!" exit; fi if (whereis maildirsync.pl | egrep "maildirsync.pl$") 2>&1 > /dev/null ; then echo "maildirsync.pl found in runpath .. Continuing to install" ; else echo "Please install maildirsync-0.5 or greater before continuing ( http://hacks.dlux.hu/maildirsync/ ). Be sure that mailsync.pl is in your runpath" ; echo "Note: You must install this package on both this slave and the master that it will sync with!!!!" exit; fi echo "You will need a set of rsa keys for ${ME} to use this software" echo "Do you want to create a key for ${ME} now? (yes or no .. If you already have one say no)" read -e A if [ "${A}" = "yes" ] ; then ssh-keygen -t rsa -q -N "" -f ${HOME}/.ssh/id_rsa fi echo "Now you need to distribute the public key to this slave host's master" echo "Would you like to transfer the key now? (yes or no .. you will need the password for ${ME} on the master)" read -e A if [ "${A}" = "yes" ] ; then echo "What is the FQDN or IP Address of the master? (yes or no)" read -e MASTER echo "HOME: ${HOME} on MASTER: ${MASTER}" scp ${HOME}/.ssh/id_rsa.pub ${MASTER}:${HOME}/.ssh/authorized_keys fi source ./mailbalance.conf ##################################################################################### # Set up remote sync environment # ##################################################################################### PROGDIRNAME=`echo ${PROGDIR} | sed -e 's/\///' | sed -e 's/.*\///g'` PROGDIRPATH=`dirname ${PROGDIR}` if (ssh ${MASTER} "ls -a ${PROGDIRPATH}" 2>/dev/null | grep "${PROGDIRNAME}" > /dev/null); then echo "${PROGDIR} exists on the master"" else echo "Creating ${PROGDIR} tree on master"" ssh ${MASTER} "mkdir -p ${PROGDIR}/lib" ; ssh ${MASTER} "mkdir ${PROGDIR}/log" ; ssh ${MASTER} "mkdir ${PROGDIR}/trashbags" fi echo '' echo "If all install routines of above questions completed, you are ready to use the software. Otherwise fix whatever broke the install and rerun the intaller." ./contrib/mailbalance-0.2.0/todo.txt0000644000175000017500000000111411463604477020033 0ustar silverdogsilverdog1. digest sanity checking to reduce overhead - done. 2. cause drsync to backup excluded-files in user trashbag dir before deleting them - ask dlux.. 3. use config file to set progdir and master, etc.. - done 4. create installer 5. break prep routines out into seperate function - done 6. write "write routines" for prepped.users.db - done 7. write function to test for exist logfile: if yes append, if no create, appends, and delete oldest (allow config var to control roll interval) 8. add verbosity levels 5. add creation of a cron job to monitor run state of mailbalance to installer ./contrib/mailbalance-0.2.0/do_ms.sh0000755000175000017500000000305511463604477017773 0ustar silverdogsilverdog #!/usr/local/bin/bash # $@ SHOULD HOLD THREE ARGS: # $1 = THE MASTER'S HOSTNAME OR IP # $2 = A USERNAME # $3 = THE PROGRAM DIRECTORY # $4 = MODE ME='' ME=`whoami` if [ $ME != "root" ] ; then echo "You must be root to run this script" exit fi source ./functions.inc HOME=/home/${2} cd $HOME PROGDIR=${3}; MDSDIGEST=MAILDIRSYNC.gz DRDIGEST=DRSYNC.bz2 MAILDIRSYNC_OPTS="-r -a md5 --backup=${PROGDIR}/trashbags/${2} --rsh-sep=, --rsh=ssh,-C" TOKEN=''; while [ "${TOKEN}" = '' ] ; do rolllog ${2} echo "Beginning to log..." echo ">>>>-----> ${2} - `date "+%H:%M:%S on %m-%d-%Y"` <-----<<<<" maildirsync.pl $MAILDIRSYNC_OPTS $1:${HOME}/Maildir ${HOME}/Maildir ${PROGDIR}/lib/${2}.${MDSDIGEST} fixperms ${1} ${2} maildirsync.pl $MAILDIRSYNC_OPTS ${HOME}/Maildir $1:${HOME}/Maildir ${PROGDIR}/lib/${2}.${MDSDIGEST} fixperms ${1} ${2} drsync.pl --verbose=2 --rsh=ssh --recursive --state-file=${PROGDIR}/lib/${2}.master.${DRDIGEST} ${HOME}/Maildir $1:${HOME} fixperms ${1} ${2} drsync.pl --verbose=2 --rsh=ssh --exclude=BACKUP --recursive --state-file=${PROGDIR}/lib/${2}.slave.${DRDIGEST} $1:${HOME}/Maildir ${HOME} fixperms ${1} ${2} drsync.pl --delete-excluded --verbose=2 --backup --rsh=ssh --exclude=BACKUP --recursive --state-file=${PROGDIR}/lib/${2}.slave.${DRDIGEST} $1:${HOME}/Maildir ${HOME} fixperms ${1} ${2} TOKEN=DONE echo "Ending log entry..." echo ">>>>-------------------------------------------------------------------------<<<<" done 2>&1 | tee -a ${PROGDIR}/log/${2}-maildirsync-`date +%Y%m`.log ./contrib/mailbalance-0.2.0/scoop_users.pl0000755000175000017500000000042611463604477021236 0ustar silverdogsilverdog#!/usr/bin/perl open(PASSWD, '/etc/passwd'); while () { chomp; ($username, $passwd, $uid, $gid, $gcos, $home, $shell) = split(/:/); if ($uid >= 1000){ if ( /\/home/ ){ $home =~ s/\/home\///g; print "$home "; } } } ./contrib/mailbalance-0.2.0/PREPPED.USERS.db0000644000175000017500000000024111463604477020633 0ustar silverdogsilverdog#this file holds username:disposition pair #it is written to and used for lookup by mailbalance #Do not modify this unless you know what you are doing test:OK ./contrib/mailbalance-0.2.0/README0000644000175000017500000000657111463604477017221 0ustar silverdogsilverdog############################################################### # mailbalance-0.2.0 # ############################################################### # Contributed to the maildirsync project by: # # Joel Holder # ############################################################### Platform: FreeBSD 4.8 Requirments: bash-2.05b perl-5.8.0_7 drsync-0.4.2 (In run path) maildirsync-0.5 (In run path) courier-imap-2.1.1 (Or just the maildirmake binary from this package in your run path) A Maildirs MTA, i.e. Postfix-2.2 or Maybe Qmail-x? Note: You may be able to use earlier version of these packages. I can only attest that these scripts work with these. Incidentally, these scripts come with absolutely no warranty, implicit or otherwise. Should you chose to try these in a production environment or with data that you care about, I suggest that you test every function of this software thoroughly in your environment before fully deploying it. Also, just so there no question about liability regarding problems that may occur as a result of your using this software, that is on you too. Finally, your usage of this software is an implicit agreement with all requirments in this document heretofore. That being said, this software will probably work on other Unix/Linux platforms, but may require some minor tweaking. Summary: This set of scripts is designed to keep system-wide Maildir mail stores in sync between servers. Logging: log directory Verbosity level has been turned up for both drsync and maildirsync, so the logs contain lots of info by default. You may want to reduce this if disk space is a concern or you do not need this level info. One month worth logs are all that are saved by default. You can modify this length by hacking the rolllog function. Transfer Digests: lib directory These are records of what has been transfered. There will be a lib directory on both sides (slave <---> master). However, only the digests where the master is the recipient are stored on the master. ALL others are in the slave host's lib. Deleted Mail: trashbags directory Every user with deleted mail on the slave will have one of these. Future revs will pull it down from the master as well. Configuration: mailbalance.conf There are only a few things to configure when setting this up. I advise checking very carefully that each requirment has been met before attempting to run mailbalance, otherwise you may cause unanticipated consequences. variable defs: $MASTER = the host running the primary maildirs mta. $PROGDIR = this directory (you may place it anywhere on the system) $ADMIN = who should get alerts from the system $FILE = the is the prepped users database file Note: Dont forget to adjust the paths to bash or perl in the following scripts: install.sh do_ms.sh mailbalance scoop_users.pl Usage: Mailbalance can be run for a single user or multiple. By default it runs in verbose/debugging mode, however if run with the -d switch, it will run in daemon mode suppressing output and forking into the background. Below are some examples: #Run for single user.. ./mailbalance blah #Run for multiple users.. ./mailbalance blah bleh bluh #Run for all users (not root).. ./mailbalance `./scoop_users.pl` #Or try: ./mailbalance < `cat userlist.txt` ./contrib/mailbalance-0.2.0/functions.inc0000644000175000017500000001247111463604477021040 0ustar silverdogsilverdog function checkuser { USER=${1} shift MATCH=`cat $FILE \ | sed -e 's/#.*//g' \ | awk "/\$USER:.*/{print $1}" \ | awk -F : '{print $2}' \ ` echo $MATCH } function fixperms { ssh ${1} "chown -R ${2} $HOME/Maildir" chown -R ${2} $HOME/Maildir } function doit { if [ "${MODE}" = "daemon" ] ; then ${PROGDIR}/do_ms.sh ${MASTER} ${1} ${PROGDIR} ${MODE} 2>&1 1>/dev/null else ${PROGDIR}/do_ms.sh ${MASTER} ${1} ${PROGDIR} ${MODE} fi } function luserprep { echo "Running local user prep routines for ${1}..." if [ -d /home/${1} ] ; then echo "HOME DIRECTORY EXISTS" else echo "NO HOME LOCAL DIRECTORY FOR USER: ${1}" echo "CREATING LOCAL HOME DIRECTORY" mkdir /home/${1} echo "NO LOCAL HOME DIRECTORY ON `hostname` FOR USER: ${1} .. CREATED IT" if (chown ${1} /home/${1}) 2> /dev/null ; then echo "CHANGED OWNERSHIP OF LOCAL HOMEDIRECTORY TO: ${1}" else echo "CANNOT CHANGE OWNERSHIP OF HOME DIRECTORY FOR USER: ${1} 2> /dev/null" echo "THIS USER PROBABLY DOES NOT EXIST ON THIS SLAVE HOST: `hostname` 2> /dev/null" notify ${1} "CANNOT CHANGE OWNERSHIP OF HOME DIRECTORY FOR USER: ${1}. THIS USER PROBABLY DOES NOT EXIST ON THIS SLAVE HOST: `hostname`" fi fi if [ -d /home/${1}/Maildir ] ; then echo "LOCAL MAILDIR EXISTS" else echo "NO LOCAL MAILDIR .. CREATING NEW MAILDIR NOW" maildirmake /home/${1}/Maildir chown -R ${1} /home/${1}/Maildir if [ -d /home/${1}/Maildir ] ; then echo "CREATED TRASHBAGS DIRECTORY FOR USER: ${1}" else echo "CANNOT CREATE Maildir FOR USER: ${1}" notify ${1} "CANNOT CREATE LOCAL USER MAILDIR" fi fi if [ -d ${PROGDIR}/trashbags/${1} ] ; then echo "TRASHBAG DIRECTORY FOR USER EXISTS" else echo "CREATING TRASHBAGS DIRECTORY FOR USER: ${1}" mkdir ${PROGDIR}/trashbags/${1} if [ -d ${PROGDIR}/trashbags/${1} ] ; then echo "CREATED TRASHBAGS DIRECTORY FOR USER: ${1}" else echo "CANNOT CREATE USER TRASHBAG DIRECTORY FOR USER: ${1}" notify ${1} "CANNOT CREATE USER TRASHBAG DIRECTORY" fi fi echo "Finished running local user prep routines for ${1}..." } function ruserprep { echo "Running remote user prep routines for ${1}..." if (ssh ${MASTER} "find /home/ -maxdepth 1 -type d -name ${1} | egrep \"${1}$\"") > /dev/null; then echo "REMOTE HOME DIRECTORY FOR THIS USER ALREADY EXISTS" else echo "NO REMOTE HOME DIRECTORY FOR THIS USER .. CREATING IT NOW" ssh ${MASTER} "mkdir /home/${1}" if (ssh ${MASTER} "find /home/ -maxdepth 1 -type d -name ${1} | egrep \"${1}$\"") > /dev/null; then echo "CREATED REMOTE HOME DIRECTORY FOR USER: ${1}" else echo "CANNOT CREATE REMOTE HOME DIRECTORY FOR USER: ${1}" notify ${1} "CANNOT CREATE REMOTE HOME DIRECTORY FOR USER: ${1}. PERHAPS I CANNOT CONNECT OR DO I HAVE RIGHTS?" fi if (ssh ${MASTER} "chown ${1} /home/${1}") 2> /dev/null ; then echo "CHANGED OWNERSHIP OF REMOTE HOMEDIRECTORY TO: ${1}" else echo "CANNOT CHANGE OWNERSHIP OF REMOTE HOME DIRECTORY FOR USER: ${1}" echo "THIS USER PROBABLY DOES NOT EXIST ON THE MASTER HOST: `hostname`" notify ${1} "CANNOT CHANGE OWNERSHIP OF REMOTE HOME DIRECTORY FOR USER: ${1}. THIS USER PROBABLY DOES NOT EXIST ON THE MASTER HOST: `ssh ${MASTER} \"hostname\"`" fi fi if (ssh ${MASTER} "find /home/${1}/ -maxdepth 1 -type d -name Maildir | egrep \"Maildir$\"") > /dev/null; then echo "REMOTE MAILDIR FOR THIS USER ALREADY EXISTS" else echo "NO REMOTE MAILDIR .. CREATING NEW MAILDIR NOW" ssh ${MASTER} "maildirmake /home/${1}/Maildir" ssh ${MASTER} "chown -R ${1} /home/${1}/Maildir" if (ssh ${MASTER} "find /home/${1}/ -maxdepth 1 -type d -name Maildir | egrep \"Maildir$\"") > /dev/null; then echo "CREATED REMOTE Maildir FOR USER: ${1}" else echo "CANNOT CREATE REMOTE Maildir FOR USER: ${1}" notify ${1} "CANNOT CREATE REMOTE Maildir FOR USER: ${1}" fi fi echo "Finished running remote user prep routines for ${1}..." } function writeok { sed "s/${1}:NOK/${1}:OK/g" ${FILE} 1> ${FILE}.tmp mv ${FILE}.tmp ${FILE} #for debugging - show if user OK or NOK #sed "s/${1}:NOK/${1}:OK/g" ${FILE} } function writenok { if (egrep "^${1}:NOK" ${FILE} 2>&1) ; then echo "USER ALREADY HAS AN ENTRY IN THE PREP DIGEST" else echo "ADDED USER ${1} TO PREP DIGEST WITH NOK STATUS" echo "${1}:NOK" >> ${FILE} fi } function notify { echo "There was a problem with $0 for user: $1 ERROR MSG: $2 " \ | mail -s "Mailbalance Alert" ${ADMIN} } function rolllog { LOGFILE="${PROGDIR}/log/${1}-maildirsync-`date +%Y%m`.log" if [ -f "${LOGFILE}" ] ; then echo '' else rm ${PROGDIR}/log/${1}-maildirsync* fi } #function setmode { # if [ "${1}" = "-d" ] ; then # shift # MODE=daemon; # echo ${MODE} # else # MODE=debug; # echo ${MODE} #fi #} ./contrib/mailbalance-0.2.0/mailbalance.conf0000644000175000017500000000023211463604477021424 0ustar silverdogsilverdog#Main configuration file PROGDIR=`pwd` MASTER=fugu ADMIN=jholder@networklogistic.com FILE=PREPPED.USERS.db #Not currently being used #SLEEP_FOR=300 ./maildirsync.pl0000755000175000017500000005760511463604477014700 0ustar silverdogsilverdog#!/usr/bin/env perl # ######################################################################### # Imports # ######################################################################### use File::Basename; use File::Copy qw(copy move); use File::Path qw(mkpath); use IO::Handle; use IPC::Open2; use Fcntl qw(SEEK_SET); use UNIVERSAL qw(isa); use strict; use warnings; require 5.006; # ######################################################################### # Constants # ######################################################################### my $VERSION = '1.2'; my $REVISION = q$Id$; my $BASENAME = basename($0); my $STATE_FILE_FIRST_LINE = "# maildirsync state file. ". "Do not edit unless you know what you are doing\n"; # long name type:default short source target my @OPTSPEC = (qw( recursive b:0 r 1 1 backup s b 0 1 backup-tree b:0 B 0 1 bzip2 s:bzip2 - 1 0 gzip s:gzip - 1 0 maildirsync s:maildirsync.pl - 1 1 mode i:0 - 0 0 rsh s:ssh R 0 0 verbose I:0 v 1 1 alg ?id|md5:id a 1 1 delete-before b:0 d 0 1 version b V 0 0 short-version b - 0 0 exclude s:[] x 1 1 exclude-source s:[] - 1 0 exclude-target s:[] - 1 0 rename s:[] N 1 0 destination ?win|lin:none - 0 0 )); push @OPTSPEC, "rsh-sep", "s: +", "-", 0, 0; use constant SOURCE_MODE => "source"; use constant TARGET_MODE => "target"; # Commands use constant DELETE_COMMAND => "DEL"; use constant NEW_COMMAND => "NEW"; use constant END_COMMANDS => "END"; use constant SEND_COMMAND => "SEND"; use constant COMMIT_COMMAND => "COMMIT"; use constant COMMIT_OK => "COMMIT_OK"; # file-data array members use constant ID => 0; use constant IDSTORE => 1; use constant DATAH => 2; # ######################################################################### # Global variables # ######################################################################### my $MODE = "startup"; our (%OPTHASH, %SHORT_OPTS, %OPT, @SOURCE_OPT, @TARGET_OPT); # ######################################################################### # Subs # ######################################################################### sub verbose ($$) { my ($verbosity_level, $message) = @_; print STDERR "$MODE: ".(" " x $verbosity_level)."$message\n" if $OPT{verbose} >= $verbosity_level; } sub add_opt ($;$) { my ($optname, $value) = @_; exit_with_error("Invalid parameter: $optname") if !$optname || !exists $OPTHASH{$optname}; my ($type, $source_opt, $target_opt) = @{ $OPTHASH{$optname} }; if ($type eq 's' || $type eq 'i') { $value = shift @ARGV if !defined $value; } elsif ($type eq 'b') { $value = 1; } elsif ($type eq 'I') { # increment $value = ($OPT{$optname} || 0)+1; } elsif (my ($regex) = $type =~ /^\?(.*)/) { $value = shift @ARGV if !defined $value; exit_with_error("Invalid parameter value: $optname: $value") if $value !~ /^$regex$/; } verbose 4 => "add option $optname = ".($value || ""); if (!isa($OPT{$optname}, 'ARRAY')) { $OPT{$optname} = $value; } else { push @{$OPT{$optname}}, $value; } push @SOURCE_OPT, "--$optname=$value" if $source_opt; push @TARGET_OPT, "--$optname=$value" if $target_opt; } sub exit_with_error ($) { my ($error) = @_; die "$error\n"; } sub source_mode ($$$$) { my ($rpipe, $wpipe, $path, $state_file) = @_; $MODE = SOURCE_MODE; my $oldfh = select $wpipe; verbose 1 => "Reading state file"; my $state = read_state_file($state_file); verbose 1 => "Reading directory structure"; my $filedata = read_filelist($path); my @old_files = sort keys %{$state->[ID] ||= {} }; verbose 1 => "Calculating digest informations on old source files"; foreach my $k (sort keys %{ $state->[ID] }) { add_store_state($state, $k, calc_store_state($path, $k, $filedata->[ID]->{$k} || $state->[ID]->{$k})) if !defined $state->[IDSTORE]->{$k} || !$state->[IDSTORE]->{$k}; } verbose 1 => "Sending change / deletion requests"; my %new_files = %{ $filedata->[ID] }; my @to_be_deleted; foreach my $k (@old_files) { if (!exists $filedata->[ID]->{$k}) { push @to_be_deleted, $k; send_command($wpipe, DELETE_COMMAND, $k); } elsif ($filedata->[ID]->{$k} ne $state->[ID]->{$k}) { send_new_command($wpipe, $state, $filedata, $k); } delete $new_files{$k}; } verbose 1 => "Calculating digest informations on new source files"; my @new_files = sort keys %new_files; foreach my $k (@new_files) { my $store_state = calc_store_state($path, $k, $filedata->[ID]->{$k}); add_store_state($state, $k, $store_state); } verbose 1 => "Sending new file requests"; foreach my $k (@new_files) { send_new_command($wpipe, $state, $filedata, $k); } send_command($wpipe, END_COMMANDS); local $|=1; my @files_to_send; verbose 1 => "Waiting for answer"; while (1) { my @cmd = receive_command($rpipe); last if $cmd[0] eq END_COMMANDS; die "Protocol error" if $cmd[0] ne SEND_COMMAND; my (undef, $fileid, $header_only) = @cmd; die "Invalid file to send" if !exists $filedata->[ID]->{$fileid}; push @files_to_send, [ $fileid, $header_only ]; } verbose 1 => "Sending files"; foreach my $filed (@files_to_send) { my ($file, $header_only) = @$filed; send_file($wpipe, $path, $file, $filedata->[ID]->{$file}, $header_only); } send_command($wpipe, COMMIT_COMMAND); my @cmd = receive_command($rpipe); if ($cmd[0] ne COMMIT_OK) { die "Cannot commit changes, bad answer from target: @cmd\n"; } verbose 1 => "Saving state file"; $state->[ID] = $filedata->[ID]; save_state_file($state_file, $state); select $oldfh; verbose 1 => "Work Finished"; close $rpipe; close $wpipe; } sub target_mode ($$$) { my ($rpipe, $wpipe, $path) = @_; $MODE = TARGET_MODE; my $oldfh = select $wpipe; verbose 1 => "Reading directory structure"; my $filedata = read_filelist($path); my @files_to_get; my @files_to_delete; verbose 1 => "Waiting for changes"; while (1) { my @cmd = receive_command($rpipe); my $command = shift @cmd; if ($command eq END_COMMANDS) { last; } elsif ($command eq DELETE_COMMAND) { my ($id) = @cmd; if (exists $filedata->[ID]->{$id}) { if ($OPT{"delete-before"}) { delete_file($path, $filedata->[ID]->{$id}) } else { push @files_to_delete, $filedata->[ID]->{$id} } } } elsif ($command eq NEW_COMMAND) { my ($id, $data, $header_size, @copy_from) = @cmd; if (exists $filedata->[ID]->{$id}) { # already exists -> move change_file($path, $filedata->[ID]->{$id}, $data) if $filedata->[ID]->{$id} ne $data; } else { # not exists my $copy_is_done; $copy_is_done ||= try_copy_body($path, $filedata->[ID]->{$_}, $data, $header_size) foreach @copy_from; push @files_to_get, [ $id, $data, ($copy_is_done ? 1 : 0)]; } } else { die "Unknown command received: $command @cmd"; } } verbose 1 => "Sending back file-requests"; foreach my $f (@files_to_get) { send_command($wpipe, SEND_COMMAND, $f->[0], $f->[2]); } send_command($wpipe, END_COMMANDS); local $|=1; verbose 1 => "Receiving files"; foreach my $f (@files_to_get) { receive_file($rpipe, $path, $f->[0], $f->[1], $f->[2]); } verbose 1 => "Committing changes"; my @cmd = receive_command($rpipe); die "Protocol error" if $cmd[0] ne COMMIT_COMMAND; delete_file($path, $_) foreach @files_to_delete; send_command($wpipe,COMMIT_OK); verbose 1 => "Work finished"; select $oldfh; } my $listfile_perms; sub read_state_file ($) { my ($filename) = @_; my $FH; my $state_data = []; $state_data->[ID] = {}; $state_data->[IDSTORE] = {}; $state_data->[DATAH] = {}; verbose 1 => "reading state file $filename"; $listfile_perms = 0666 ^ umask(); return $state_data if ! -e $filename; if ($filename =~ /\.bz2/) { my $pid = open $FH, "-|"; if (!$pid) { # child exec($OPT{bzip2}, "-cd", $filename); exit(1); } } elsif ($filename =~ /\.gz$/) { my $pid = open $FH, "-|"; if (!$pid) { # child exec($OPT{gzip}, "-cd", $filename); exit(1); } } else { open $FH, $filename }; return $state_data if !$FH; # no state-file $listfile_perms = (stat($filename))[2] & 07777; my $first_line = <$FH>; exit_with_error("Invalid state file header: $first_line") if $first_line ne $STATE_FILE_FIRST_LINE; while (<$FH>) { chomp; next if /^$/; my ($id, $state, $store_state) = split /\t/; exit_with_error("Invalid line in the state file: $_") if !$id; my $dirname = dirname($state); if (!match_excludes($dirname)) { $state_data->[ID]->{$id} = $state; add_store_state($state_data, $id, $store_state); } } close $FH; return $state_data; } sub read_directory ($$$;$) { my ($basepath, $path, $aref, $mailsubdir) = @_; my $dirpath = "$basepath$path"; if (match_excludes($path)) { verbose 3 => "Excluding directory: $dirpath"; return; } verbose 3 => "Reading directory: $dirpath"; opendir DIR, $dirpath or return; my @entries = readdir(DIR); closedir DIR; foreach my $e (@entries) { my $entry = "$dirpath/$e"; next if $e =~ /^\.(\.)?$/; # . , .. if (!$mailsubdir && -d $entry) { next if $e eq 'tmp'; my $newmailsubdir = $e eq 'new' || $e eq 'cur'; if ($newmailsubdir || $OPT{recursive}) { no warnings; read_directory($basepath, "$path/$e", $aref, $newmailsubdir); } } if ($mailsubdir && -f $entry) { my ($key, $filedata) = pack_filedata("$path/$e"); if ($key) { # valid file verbose 1 => "Duplicated file id entry: $key" if exists $aref->[ID]->{$key}; $aref->[ID]->{$key} = $filedata; } } } } sub read_filelist ($) { my ($path) = @_; my @file_data; $file_data[ID] = {}; read_directory($path, "", \@file_data); return \@file_data; } sub send_command ($@) { my ($channel, @command) = @_; verbose 4 => "sending command: @command"; print $channel join("\t", @command)."\n" or exit_with_error("Cannot send command: @command: $!"); } sub send_file_data ($$$) { my ($channel, $file_name, $header_only) = @_; my $FILE; if (open $FILE, $file_name) { my $file_header = read_header($FILE); my $size_to_send = length($file_header); $size_to_send = (-s $FILE) if !$header_only; print $channel $size_to_send."\n" or exit_with_error("Cannot send size header"); print $channel $file_header; if (!$header_only) { print or exit_with_error("Cannot send file header") while <$FILE>; } close $FILE; } else { print "-1\n"; } } sub receive_file_data ($$$) { my ($channel, $file_name, $header_only) = @_; my $length = <$channel>; chomp $length; my $data; my $FILE; return 0 if $length == -1; verbose 5 => "File length: $length"; mkdir_for_target_file($file_name); my $opened = open $FILE, ($header_only ? "+<" : ">"), $file_name; seek($FILE, 0, SEEK_SET) if $header_only && $opened; warn "Cannot open file $file_name for writing: $!" if !$opened; while ($length >0) { my $bytes_read = read $channel, $data, ($length > 4096 ? 4096 : $length) or exit_with_error("Cannot receive file (length: $length)"); print $FILE $data if $opened; $length -= $bytes_read; } close $FILE if $opened; return 1; } sub receive_command ($) { my ($channel) = @_; my $command = <$channel>; defined($command) or exit_with_error("Cannot read command from pipe"); chomp $command; my @command = split /\t/, $command; verbose 4 => "command received: @command"; return @command; } sub save_state_file ($$) { my ($filename, $statedata) = @_; my $FH; my $newfilename = $filename.".new.$$"; my $statedir = dirname($newfilename); unless (-d $statedir) { mkpath($statedir) or exit_with_error("Cannot create directory for state file: $statedir: $!"); verbose 3 => "created directory: $statedir"; } if ($filename =~ /\.bz2$/) { my $pid = open $FH, "|-"; if (!$pid) { # child open STDOUT, ">" ,$newfilename or exit 1; exec($OPT{bzip2}); exit(1); } } elsif ($filename =~ /\.gz$/) { my $pid = open $FH, "|-"; if (!$pid) { # child open STDOUT, ">", $newfilename or exit 1; exec($OPT{gzip}); exit(1); } } else { open $FH, ">", $newfilename or $FH = undef }; exit_with_error("Cannot open temporary state file for writing: $newfilename") if !$FH; print $FH $STATE_FILE_FIRST_LINE; print $FH join("\t",$_, $statedata->[ID]->{$_}, ($statedata->[IDSTORE]->{$_} || ""))."\n" foreach keys %{$statedata->[ID]}; close $FH; chmod $listfile_perms, $newfilename or exit_with_error("Cannot chmod temporary state file: $!"); if (-f $filename) { move $filename, $filename."~" or exit_with_error("Cannot make backup of $filename: $!"); } move $newfilename, $filename or exit_with_error("Cannot move temporary state file $filename: $!"); if (-f $filename."~") { unlink $filename."~" or exit_with_error("Cannot unlink backup state file: $filename: $!"); } } my $backup_dir_created = 0; sub delete_file ($$) { my ($basepath, $filedata) = @_; my ($path) = unpack_filedata($filedata); if ($OPT{backup}) { my $dest = $OPT{backup}; if ($OPT{'backup-tree'}) { $dest .= $path; mkdir_for_target_file($dest); } else { mkpath($dest) if !$backup_dir_created++; } verbose 2 => "Deleting file: $path (moving to backup directory)"; move "$basepath$path", $dest or warn "Cannot move $path to the backup directory: $!\n" ; } else { verbose 2 => "Deleting file: $path"; unlink "$basepath$path" or warn "Cannot unlink $path: $!\n" } } sub change_file ($$$) { my ($basepath, $filedata1, $filedata2) = @_; my ($path1) = unpack_filedata($filedata1); my ($path2) = unpack_filedata($filedata2); mkdir_for_target_file("$basepath$path2"); verbose 2 => "Move file from $path1 to $path2"; move "$basepath$path1", "$basepath$path2" or warn "Cannot move $path1 to $path2: $!\n"; } sub send_file ($$$$$) { my ($wpipe, $basepath, $file, $filedata, $header_only) = @_; my ($path) = unpack_filedata($filedata); verbose 2 => "Sending file".($header_only ? " (header only)" : "").": $path"; send_file_data($wpipe, "$basepath$path", $header_only); } sub receive_file ($$$$$) { my ($rpipe, $basepath, $file, $filedata, $header_only) = @_; my ($path) = unpack_filedata($filedata); verbose 2 => "Receiving file".($header_only ? " (header only)" : "").": $path"; my $target_name = "$basepath$path"; if($OPT{destination} eq 'win') {$target_name =~ s/\:/:/g;} elsif($OPT{destination} eq 'lin') {$target_name =~ s/:/\:/g;} my $temp_name = $target_name; $temp_name =~ s{^(.*)(?:new|cur)(/.*)$}{$1tmp/$2}; if($OPT{destination} eq 'win') {$temp_name =~ s/\:/:/g;} elsif($OPT{destination} eq 'lin') {$temp_name =~ s/:/\:/g;} receive_file_data($rpipe, $temp_name, $header_only) or return; # No files received: nothing to do mkdir_for_target_file($target_name); rename $temp_name, $target_name or warn "Cannot rename the temporary file $temp_name to target $target_name"; } sub unpack_filedata ($) { my ($filedata) = @_; return ($filedata); } sub pack_filedata ($) { my ($path) = @_; my ($message_id) = $path =~ m{.*/([^\.\:/][^\:/]*?)(?:(?:\:|:)(?:1|2),[^/]*)?$} or return (); # not valid $message_id =~ m{:} and return (); # not valid return ($message_id, "$path"); } my %MKDIR_HASH; sub mkdir_for_target_file ($) { my ($filename) = @_; my $dirname = dirname($filename); $dirname =~ s/(new|cur|tmp)\/?$//; no warnings; return if $MKDIR_HASH{$dirname}++; verbose 3 => "Creating directory tree: $filename"; mkpath($dirname."/new"); mkpath($dirname."/tmp"); mkpath($dirname."/cur"); } sub add_store_state ($$$) { my ($state_data, $id, $store_state) = @_; return if !$store_state; my ($header_data, $data_hash) = unpack_store_state($store_state); $state_data->[IDSTORE]->{$id} = $store_state; push @{ $state_data->[DATAH]->{$data_hash} }, $id; } sub unpack_store_state ($) { my ($store_state) = @_; return $store_state =~ /^(.*)-(.*)$/; } sub calc_store_state ($$$) { my ($basepath, $id, $filedata) = @_; return undef if $OPT{alg} ne "md5"; my ($path, undef) = unpack_filedata($filedata); my $md5 = Digest::MD5->new; open my $FH, "$basepath$path" or return undef; my $str = read_header($FH); my $header_data = length($str); $md5->addfile($FH); my $data_hash = $md5->hexdigest.((-s $FH) - length($str)); my $return_data = "$header_data-$data_hash"; close $FH; verbose 2 => "Calculated data for file $id: $return_data"; return $return_data; } sub send_new_command ($$$$) { my ($wpipe, $state, $filedata, $k) = @_; my ($header_size, $data, %datahash) = (0); if (my $hash = $state->[IDSTORE]->{$k}) { ($header_size, $data) = unpack_store_state($hash); $datahash{$_} = 1 foreach @{ $state->[DATAH]->{$data} || [] }; } delete $datahash{$k}; send_command($wpipe, NEW_COMMAND, $k, rename_file_in_filedata($filedata->[ID]->{$k}), $header_size, keys %datahash); } sub read_header { my ($FH) = @_; my $str = ""; while (<$FH>) { $str .= $_; last if /^$/; } return $str; } sub try_copy_body ($$$$) { my ($basepath, $source_data, $target_data, $header_size) = @_; return if !$source_data; my ($source_path, undef) = unpack_filedata($source_data); return if $header_size == 0; verbose 3 => "Trying to copy body from message: $source_path, header_size: $header_size"; my $FILE; open $FILE, "$basepath$source_path" and do { # the source file exists read_header($FILE); # we skip the original header my ($target_path, undef) = unpack_filedata($target_data); my $target_temp_file = "$basepath$target_path"; $target_temp_file =~ s{^(.*)(?:new|cur)(/.*)$}{$1tmp/$2}; mkdir_for_target_file($target_temp_file); open my $OFILE, ">$target_temp_file" or do { warn "Cannot open temp file for output"; return 0 }; seek($OFILE, $header_size, SEEK_SET); my ($buffer, $bytes_read); while (($bytes_read = read($FILE, $buffer, 4096)) > 0) { print $OFILE substr($buffer, 0, $bytes_read); } if (!defined $bytes_read) { warn "Cannot copy source file $source_path to $target_temp_file: $!\n"; return 0 } verbose 2 => "File body for $target_path is copied from $source_path"; close $OFILE; close $FILE; return 1; }; return 0; } sub match_excludes ($) { my ($path) = @_; my $local_source = $MODE eq SOURCE_MODE ? $OPT{"exclude-source"} : $OPT{"exclude-target"}; foreach my $exclude (@{ $OPT{exclude} }, @$local_source) { return 1 if $path =~ /$exclude/; } return 0; } sub rename_file_in_filedata ($) { my ($filedata) = @_; ($_) = unpack_filedata($filedata); foreach my $rename (@{ $OPT{rename} }) { eval $rename; exit_with_error("Error running command '$rename' on '$_'. Error: '$@'") if $@; } $filedata = pack_filedata($_); } # ######################################################################### # Main program # ######################################################################### while (@OPTSPEC) { my $optname = shift @OPTSPEC; my ($type, $default) = shift(@OPTSPEC) =~ /^(.+?)(?:\:(.*))?$/; my $shortname = shift @OPTSPEC; my $source_opt = shift @OPTSPEC; my $target_opt = shift @OPTSPEC; $OPTHASH{$optname} = [$type, $source_opt, $target_opt]; if (defined $default && $default eq '[]') { $OPT{$optname} = []; } else { $OPT{$optname} = $default; } $SHORT_OPTS{$shortname} = $optname if $shortname ne '-'; } while (@ARGV) { my $arg = shift @ARGV; last if $arg eq '--'; if (my ($optname, $optval) = $arg =~ /^--([\w-]+)(?:\=(.*))?/) { add_opt($optname, $optval); } elsif (my ($short_opts) = $arg =~ /^-(\w+)/) { add_opt($SHORT_OPTS{$_}) foreach split //, $short_opts; } else { unshift @ARGV, $arg; last; } } if ($OPT{version}) { print "$BASENAME version $VERSION, revision: $REVISION\n\n"; print "Type perldoc $BASENAME for help\n\n"; exit 0; } if ($OPT{"short-version"}) { print "$VERSION\n"; exit 0; } # managing the source and target modes $SIG{PIPE} = sub { }; if ($OPT{alg} eq 'md5') { eval { require Digest::MD5 }; exit_with_error("Digest::MD5 module is required for md5 algorithm") if !$INC{"Digest/MD5.pm"}; } verbose 1 => "Starting source and target modes"; if ($OPT{mode} eq 'source') { # source pipe mode my ($srcpath, $state) = @ARGV; source_mode(\*STDIN, \*STDOUT, $srcpath, $state); } elsif($OPT{mode} eq 'target') { # target pipe mode my ($trgpath) = @ARGV; target_mode(\*STDIN, \*STDOUT, $trgpath); } else { exit_with_error("Usage: $BASENAME [options] src target state-file") if @ARGV != 3; my ($src, $trg, $state_file) = @ARGV; my ($srchost, $srcpath) = $src =~ /^(?:(.*?)\:)?(.*)/; my ($trghost, $trgpath) = $trg =~ /^(?:(.*?)\:)?(.*)/; my @rsh_command = split /$OPT{"rsh-sep"}/, $OPT{rsh}; # verbose 0 => "Rsh command: ".join(",",@rsh_command); if (defined $srchost && defined $trghost) { exit_with_error("Source or destination must be local"); } elsif (defined $srchost) { my ($pipei, $pipeo); open2($pipei, $pipeo, @rsh_command, $srchost, $OPT{maildirsync}, "--mode=source", @SOURCE_OPT, $srcpath, $state_file); target_mode($pipei, $pipeo, $trgpath); } elsif (defined $trghost) { my ($pipei, $pipeo); open2($pipei, $pipeo, @rsh_command, $trghost, $OPT{maildirsync}, "--mode=target", @TARGET_OPT, $trgpath); source_mode($pipei, $pipeo, $srcpath, $state_file); } else { pipe(\*P1A, \*P1B); pipe(\*P2A, \*P2B); my $oldfh = select(P1B); $|=1; select(P2B); $|=1; select($oldfh); if (fork()) { source_mode(\*P1A, \*P2B, $srcpath, $state_file); } else { target_mode(\*P2A, \*P1B, $trgpath); exit 0; } } } ./maildirsync.pod0000644000175000017500000003161611463604477015036 0ustar silverdogsilverdog=head1 NAME maildirsync - Online synchronizer for Maildir-format mailboxes =head1 SYNOPSIS maildirsync.pl [ --recursive ] [ --backup path ] [ --backup-tree ] \ [ --bzip2=bzip2 ] [ --gzip=gzip ] [ --maildirsync=maildirsync.pl ] \ [ --rsh=ssh ] [ --verbose ] [ --alg=md5 ] [ --delete-before ] \ [ --exclude=^/Folder1 ] [ --exclude=^/Fold.*er2 ] \ [ --exclude-source=^/Folder3 ] [ --exclude-target=^/Folder4 ] \ [ --rename="s/SourceFolder/TargetFolder/" ] \ [ --destination=win|lin ] \ [ -r ] [ -b path ] [ -B ] [ -R ssh ] [ -v ] [ -a ] [ -d ] \ source dest state_file.bz2 A simple two-way synchronization: maildirsync.pl -rvv -a md5 desktop:Maildir Maildir lib/sync_desk_note.bz2 maildirsync.pl -rvv -a md5 Maildir desktop:Maildir lib/sync_note_desk.bz2 =head1 DESCRIPTION maildirsync is a utility for online Maildir-synchronization. It is designed to be used on live maildir folders, be fail-safe and optimized for minimal bandwidth. =head1 HOW IT WORKS If you call the program once, it will propagate the changes from the source side to the target side. Two-way synchronization requires two state-file and two call for the program. This propagation is basically two different operations from the source side: =over 4 =item * New file is created or an existing file is moved to a new location (e.g flags are changed). =item * A file is deleted in the source side. (Will be deleted in the target side also). =back At the first phase, the source side reads the state file (which stores the state of the last synchronization) and compares it to the current state. It collects the changes and sends them the target side. The target side checks every file, which is marked new in the source file, and decides if: =over 4 =item * The file needs to be downloaded fully =item * Its header needs to be downloaded =item * We have this file already, so we need no data. =back After it decides, it sends back the requests for new files. Then the source side will send the files to the target side, which stores them into the Maildir structure. After the send operation is completed, both operation agrees upon the commit operation. Then the source side saves the new state into the state file. Note if we forget saving the state, or the program exits before it, the operation can be restarted without data loss and inconsistency, because all operations can be redone without errors. =head1 REMOTE OPERATION Maildir can be used in remote mode, so it can synchronize Maildir folders between computers. If you want to use it this way, you have to provide the host name before either the source or the target, like: maildirsync.pl ... desktop:Maildir Maildir lib/maildirsync.bz2 or maildirsync.pl ... Maildir desktop:Maildir lib/maildirsync.bz2 In remote mode, the target side must have maildirsync installed also. (See the --maildirsync command-line argument). The state-file must be in the same system as the source, so the source file in the first example is searched in the "desktop" computer, and in the local computer in the second example. At least source or the destination must be local, so you cannot sync maildirs on two different remote hosts. =head1 COMMAND-LINE SWITCHES Some command line switches has two form: a short form and a longer form. In the short form, the switches can be grouped, like: -vvvr. Short options with parameters also can be grouped, but the parameter must be the following command-line argument, like: maildirsync.pl -rvvvbR Maildir/Trash/cur ssh ... It is the same as: maildirsync.pl --recursive --verbose --verbose --verbose --backup \ Maildir/Trash/cur --rsh ssh ... Long options can use '=' for assigning the parameter, or they can use the syntax above. Let's see what switches we have: =over 4 =item --recursive, -r Process the base folder as the recursive collection of Maildir folders. =item --backup dir, -b dir The deleted files are backed up to the specified directory (this directory does not needs to be a maildir folder). The directory is relative to the start-up directory, not the target base folder! The directory is created if it does not exists. =item --backup-tree, -B This option is only useful when used in conjunction with the --backup option. If set, deleted messages are moved to a Maildir folders tree inside the backup directory with the same relative path. The resulting backup directory can be used with any Maildir-capable application (MUAs, MTAs, etc.). =item --bzip2 bzip2 Path to the bzip2 utility (used only when the state file has .bz2 extension). Note that using bzip2 is turned out to be quite unstable, it sometimes leave the state file empty or corrupted. =item --gzip gzip Path to the gzip utility (used only when the state file has .gz extension). =item --maildirsync maildirsync.pl Path to the maildirsync utility on the remote machine (if we use maildirsync in remote mode). =item --rsh ssh Path to the utility, which can be used to connect to the remote side. It defaults to "ssh". Note that the protocol, which is used in remote mode, does not contain compression, but the data can be compressed well, so I suggest using the ssh compression for this purpose. =item --destination If it is provided, then it does transformation in the file names. The transformations are the following: =over 4 =item * win: Converts ":" to ":" (since windows don't allow colon in the filename). =item * lin: Converts ":" to ":". =back =item --verbose, -v Adds more verbosity to the operation. There are currently 6 different verbosity level: =over 4 =item 0 No information at all. =item 1 Main operations. =item 2 Files sent, received, deleted, moved, md5 calculations. =item 3 Directories read and created. =item 4 Options + command echo. =item 5 Misc info about file transfer. =back =item --alg md5, -a md5 Selects the synchronization algorithm. Currently two algorithms are provided: =over 4 =item id (default) The messages are synchronized only by the ID of a message (the ID can be determined by the message filename). =item md5 This method is recommended for low bandwidth operations. This mode can reduce the file transfers by checking the message moves on the target side. This mode requires an MD5 sum on the body of the message, so the first use of this mode can be quite time-consuming on both sides. =back For more information about the algorithms, read the chapter about that. =item --delete-before, -d Normally the delete operations on the target side is done after the transfers. Use this switch if you don't have too much space on your hard disk. Note that using this switch can reduce the chance of detecting the message moves. =item --exclude, -x Excludes a directory regexp from the transfer and removes it from the state file. This option can be used more than once to specify more than one directories. Note, that the directory, which is matched against these regexps are the relative path of the folders with a leading '/', so if you want to exclude your Trash folder in the root of your synchronization, then you have to use the following form: --exclude=^/Trash The excluded paths are used in either source or the target side also. So if you exclude a very large directory, you will notice speedup in the source and target side also. Note that this regexp is matched for every directory that is read from the filesystem, and every directory what is found in the state file. So if you provide the exclude pattern as ^/Trash$, then it will skip the Trash directory when traversing the directory structure, but it WON'T skip files from the Trash/cur directory when reading from the state file! So be careful if you use the exclude pattern with existing state file! =item --exclude-source, --exclude-target These parameters can be used to exclude files only from one side of the synchronization. Can be useful with the rename options (below). =item --rename Can be used to rename files when transferred from one side to another. A perl expression is the parameter for this. If you use this option, use it with care, because you have to provide exactly the opposite of the name-transformation if you sync to the other side, for example: In one side, you can use (A to B): --rename="s{^/Saved/}{/ToBeSaved/}" \ --exclude-target="^/Saved/" \ In the other side, you can use (B to A): --exclude-source="^/Saved/" \ --rename="s{^/ToBeSaved/}{/Saved/}" \ In this case, the Saved folder in the A side will be synchronized with the ToBeSaved folder in the B side, and the Saved folder in the B side will be excluded from the synchronization. This scenario can be used when you don't want to store your emails in the server, but you want to use the "Saved" folder in the server too. In this case, the emails will be downloaded from the server (A side) to your laptop (B side), then you can move them to the Saved folder in B with a script. If this is done, then you can resynchronize, then the saved files will disappear from the server also. =back =head1 ALGORITHMS Currently the program has two algorithms, which can be used for synchronization. =over 4 =item id (default) This algorithm is based on the message-id-s of the messages. It assumes that a message-id exists only once in both repositories with the same id. The id can be determined form the filename. With this algorithm, you can trace the flag changes or the deletion of a message and these changes can be propagated to the other side also. It also handles if a message is copied from the "new" directory to the "cur" directory without retransmit the files over the network. This algorithm is recommended if you want a simple and quite fast operation, and if you have a not-so-slow internet connection. =item md5 This algorithm does further calculations. It stores the size of the header and the md5 sum of the message body for each message besides the data that the "id" algorithm stores. By using this algorithm, you can track the copies and moves of a message, so you don't need to retransmit large files if you move them to a new folder. If you copy or move a message from one folder to another, the header is sometimes changed by the mail-reader program. This is why we cannot simply calculate the MD5 sum of the whole message. A new message in a folder will have a new identifier also, so it doesn't violate the law that a message-id is unique. When you copy or move a message from your INBOX to your "Save" folder (for example), the new message is analyzed in the source side, header-size and md5 sum is calculated on the new message, and from the md5-hash, the source side can tell the target side what messages has the same hash-value, so the target side can copy the body from the other message. If the target side has successfully copied the body from one of those provided messages, then only the header needs to be transmitted across the network. If the target-side did not find the messages, then it requests for the body also. =back =head1 ONLINE OPERATION Online operation means that the software can be used in an online mailbox also. It assumes that the Maildir folder can be changed when the program is working, so it tries to be as fail-safe as can be. Every new file is opened in the "tmp" directory, and moved to the target place only when the file is fully downloaded. This mode of operation was the first priority, because this feature is missing from most synchronizer software, including my "drsync" utility also. =head1 SPEED I am using this program to synchronize my Mailboxes. I have 9700 emails in my mailbox and the state file (bzipped) is 283K. The first time of a two-way synchronization between a P166 server and a PIII/1200 notebook over a Cable network, where the starting position is an already synchronized directory, tooks about 10 minutes. This time is used for md5-calculation and message-id propagation. The next two-way run tooks about 40 seconds. These things are measured in Debian GNU/Linux testing/unstable operating system (08 Oct 2002). These are only the overhead of the software, not the real transfer. If you got a very big email, it needs to be transferred at least one time on the network. But if you have it in both sides, then it does not require any more transfer if you save it to different folders. =head1 TODO I am currently happy with this feature set, but if I have time, I will implement these features into the software. Anyway if you have time and willingness, I accept patches also: =over 4 =item * Message-size limit. By limiting the maximum transmitted message, you can effectively use this software if you have very low bandwidth. =item * Config-file and cron-safe operation. =back =head1 COPYRIGHT Copyright (c) 2000-2010 Szabó, Balázs (dLux) All rights reserved. This program is free software; you can redistribute it and/or modify it under the same terms as Perl itself. =head1 AUTHOR dLux (Szabó, Balázs) ./Changes0000644000175000017500000000374511463604477013311 0ustar silverdogsilverdogRevision history for the maildirsync utility: 1.2 Mon Nov 1 19:22:19 CET 2010 - New option: --backup-tree. (gpiero) - Allow spaces in the homedir path in the provided sample script. (gpiero) - Added mac and macports compatibility. (dlux) - New option: --destination to support maildirs on windows as well. (mmarini & fabrizio.sotgiu) - Create directory path for state file if needed. (gpiero) http://bugs.debian.org/379091 - Exit if cannot read command from pipe. (gpiero) http://bugs.debian.org/440364 - Support more wide range of filenames in maildir. (gpiero) http://bugs.debian.org/356207 - Typo fixes. (gpiero) - Do not use a world writable dir for logging from the provided sample script. Closes: CVE-2008-5150. (gpiero) - Use perl warnings pragma instead of -w option. (gpiero) 1.1 Wed Dec 22 22:08:20 CET 2004 - Added --exclude-source and --exclude-target and rename option 1.0 Sun Jul 11 21:23:03 CEST 2004 - Added --exclude option 0.6 Sat Dec 6 11:47:18 CET 2003 - Added a contributed software, which demonstrates the usage of maildirsync: mailbalance, a collection of scripts, which synchronizes several mailboxes between two hosts. 0.5 Mon Sep 22 08:50:02 CEST 2003 - Bug fix for initial synchronization (Can't use an undefined value as a HASH reference at ./maildirsync.pl line 115) - Better error handling on temporary file open - A sample wrapper script added to the archive - Source control moved to subversion 0.4 Fri May 9 22:35:50 CEST 2003 - File name default changed from maildirsync to maildirsync.pl to match the filename in the distribution - It now runs on perl 5.6 also - Added the possibility to parameterize the "rsh" command (--rsh-sep parameter) 0.3 Sun Mar 16 17:23:47 CET 2003 - Fix the regex, which allows full hostname in the mail-filenames. 0.2 Tue Oct 8 22:10:31 CEST 2002 - Verbosity levels are defined. - Several bugs are fixed. - Documentation updates 0.1 Mon Oct 7 00:49:40 CEST 2002 - Initial release