1 rizwank 1.1 #!/usr/bin/perl
2 #-------------------------------------------------------
3 # Convert a mail log file to a common log file for analyzing with any log
4 # analyzer.
5 #-------------------------------------------------------
6 # Tool built from original work of Odd-Jarle Kristoffersen
7 # Note 1: QMail must log in syslog format for timestamps to work.
8 # Note 2: QMail logging is not 100% accurate. Some messages might
9 # not be logged correctly or completely.
10 #
11 # A mail received to 2 different receivers, report 2 records.
12 # A mail received to a forwarded account is reported as to the original receiver, not the "forwarded to".
13 # A mail locally sent to a local alias is reported as n mails to all addresses of alias.
14 #-------------------------------------------------------
15 use strict;no strict "refs";
16
17
18 #-------------------------------------------------------
19 # Defines
20 #-------------------------------------------------------
21 use vars qw/ $REVISION $VERSION /;
22 rizwank 1.1 $REVISION='$Revision: 1.31 $'; $REVISION =~ /\s(.*)\s/; $REVISION=$1;
23 $VERSION="1.2 (build $REVISION)";
24
25 use vars qw/
26 $DIR $PROG $Extension
27 $Debug
28 %mail %qmaildelivery
29 $help
30 $mode $year $lastmon $Debug
31 $NBOFENTRYFOFLUSH
32 $MailType
33 %MonthNum
34 /;
35 $Debug=0;
36 $NBOFENTRYFOFLUSH=16384; # Nb or records for flush of %entry (Must be a power of 2)
37 $MailType=''; # Mail server family (postfix, sendmail, qmail)
38 %MonthNum = (
39 'Jan'=>1,
40 'Feb'=>2,
41 'Mar'=>3,
42 'Apr'=>4,
43 rizwank 1.1 'May'=>5,
44 'Jun'=>6,
45 'Jul'=>7,
46 'Aug'=>8,
47 'Sep'=>9,
48 'Oct'=>10,
49 'Nov'=>11,
50 'Dec'=>12
51 );
52
53
54 #-------------------------------------------------------
55 # Functions
56 #-------------------------------------------------------
57
58 sub error {
59 print "Error: $_[0].\n";
60 exit 1;
61 }
62
63 sub debug {
64 rizwank 1.1 my $level = $_[1] || 1;
65 if ($Debug >= $level) {
66 my $debugstring = $_[0];
67 if ($ENV{"GATEWAY_INTERFACE"}) { $debugstring =~ s/^ /   /; $debugstring .= "<br>"; }
68 print localtime(time)." - DEBUG $level - $. - : $debugstring\n";
69 }
70 0;
71 }
72
73 sub CleanVadminUser { $_=shift||'';
74 s/[#<|>\[\]]//g; # Remove unwanted characters first
75 s/^(.*?)-//gi; # Strip off unixuser- at beginning
76 return $_;
77 }
78
79 sub CleanEmail { $_=shift||'';
80 s/[#<|>\[\]]//g; # Remove unwanted characters first
81 return $_;
82 }
83
84 # Clean host addresses
85 rizwank 1.1 # Input: "servername[123.123.123.123]", "servername [123.123.123.123]"
86 # "root@servername", "[123.123.123.123]"
87 # Return: servername or 123.123.123.123 if servername is 'unknown'
88 sub CleanHost {
89 $_=shift||'';
90 if (/^\[(.*)\]$/) { $_=$1; } # If [ip] we keep ip
91 if (/^unknown\s*\[/) { $_ =~ /\[(.*)\]/; $_=$1; } # If unknown [ip], we keep ip
92 else { $_ =~ s/\s*\[.*$//; }
93 $_ =~ s/^.*\@//; # If x@y, we keep y
94 return $_;
95 }
96
97 # Return domain
98 # Input: host.domain.com, <user@domain.com>, <>
99 #
100 sub CleanDomain { $_=shift;
101 s/>.*$//; s/[<>]//g;
102 s/^.*@//;
103 if (! $_) { $_ = 'localhost'; }
104 return $_;
105 }
106 rizwank 1.1
107 # Return string without starting and ending space
108 #
109 sub trim { $_=shift;
110 s/^\s+//; s/\s+$//;
111 return $_;
112 }
113
114 # Write a record
115 #
116 sub OutputRecord {
117 my $year=shift;
118 my $month=shift; # Jan,Feb,... or 1,2,3...
119 my $day=shift;
120 my $time=shift;
121 my $from=shift;
122 my $to=shift;
123 my $relay_s=shift;
124 my $relay_r=shift;
125 my $code=shift;
126 my $size=shift||0;
127 rizwank 1.1 my $forwardto=shift;
128 my $extinfo=shift||'-';
129
130 # Clean day and month
131 $day=sprintf("%02d",$day);
132 $month=sprintf("%02d",$MonthNum{$month}||$month);
133
134 # Clean from
135 $from=&CleanEmail($from);
136 $from||='<>';
137
138 # Clean to
139 if ($mode eq 'vadmin') { $to=&CleanVadminUser($to); }
140 else { $to=&CleanEmail($to); }
141 $to||='<>';
142
143 # Clean relay_s
144 $relay_s=&CleanHost($relay_s);
145 $relay_s||=&CleanDomain($from);
146 $relay_s=~s/\.$//;
147 if ($relay_s eq 'local' || $relay_s eq 'localhost.localdomain') { $relay_s='localhost'; }
148 rizwank 1.1
149 # Clean relay_r
150 $relay_r=&CleanHost($relay_r);
151 $relay_r||="-";
152 $relay_r=~s/\.$//;
153 if ($relay_r eq 'local' || $relay_r eq 'localhost.localdomain') { $relay_r='localhost'; }
154 #if we don't have info for relay_s, we keep it unknown, awstats might then guess it
155
156 # Write line
157 print "$year-$month-$day $time $from $to $relay_s $relay_r SMTP $extinfo $code $size\n";
158
159 # If there was a redirect
160 if ($forwardto) {
161 # Redirect to local address
162 # TODO
163 # Redirect to external address
164 # TODO
165 }
166 }
167
168
169 rizwank 1.1
170 #-------------------------------------------------------
171 # MAIN
172 #-------------------------------------------------------
173
174 # Prepare QueryString
175 my %param=();
176 for (0..@ARGV-1) { $param{$_}=$ARGV[$_]; }
177 foreach my $key (sort keys %param) {
178 if ($param{$key} =~ /(^|-|&)debug=([^&]+)/i) { $Debug=$2; shift; next; }
179 if ($param{$key} =~ /^(\d+)$/) { $year=$1; shift; next; }
180 if ($param{$key} =~ /^(standard|vadmin)$/i) { $mode=$1; shift; next; }
181 }
182 if ($mode ne 'standard' and $mode ne 'vadmin') { $help = 1; }
183
184 ($DIR=$0) =~ s/([^\/\\]*)$//; ($PROG=$1) =~ s/\.([^\.]*)$//; $Extension=$1;
185
186 my $starttime=time();
187 my ($nowsec,$nowmin,$nowhour,$nowday,$nowmonth,$nowyear,$nowwday,$nowyday) = localtime($starttime);
188 $year||=($nowyear+1900);
189
190 rizwank 1.1 # Show usage help
191 if ($help) {
192 print "----- $PROG $VERSION -----\n";
193 print <<HELPTEXT;
194 $PROG is mail log preprocessor that convert a mail log file (from
195 postfix, sendmail or qmail servers) into a human readable format.
196 The output format is also ready to be used by a log analyzer, like AWStats.
197
198 Usage:
199 perl maillogconvert.pl [standard|vadmin] [year] < logfile > output
200
201 The first parameter specifies what format the mail logfile is :
202 standard - logfile is standard postfix,sendmail,qmail or mdaemon log format
203 vadmin - logfile is qmail log format with vadmin multi-host support
204
205 The second parameter specifies what year to timestamp logfile with, if current
206 year is not the correct one (ie. 2002). Always use 4 digits. If not specified,
207 current year is used.
208
209 If no output is specified, it goes to the console (stdout).
210
211 rizwank 1.1 HELPTEXT
212 sleep 1;
213 exit;
214 }
215
216 #
217 # Start Processing Input Logfile
218 #
219 $lastmon=0;
220 my $numrecord=0;
221 my $numrecordforflush=0;
222 while (<>) {
223 chomp $_; s/\r//;
224 $numrecord++;
225 $numrecordforflush++;
226
227 my $mailid=0;
228
229 if (/^__BREAKPOINT__/) { last; } # For debug only
230
231 ### <CJK> ###
232 rizwank 1.1 my ($mon)=m/(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s/;
233 if ($mon) {
234 $mon = $MonthNum{$mon};
235 if ($mon==12 && $lastmon==1 ){$year--;}
236 if ($mon==1 && $lastmon==12){$year++;}
237 $lastmon=$mon;
238 }
239 ### </CJK> ###
240
241 if (/^#/) {
242 debug("Comment record");
243 next;
244 }
245
246 #
247 # Get sender host for postfix
248 #
249 elsif (/: client=/) {
250 $MailType||='postfix';
251 # Example:
252 # postfix: Jan 01 07:27:32 apollon.com postfix/smtpd[1684]: 2BC793B8A4: client=remt30.cluster1.abcde.net[209.225.8.40]
253 rizwank 1.1 my ($id,$relay_s)=m/\w+\s+\d+\s+\d+:\d+:\d+\s+[\w\-\.\@]+\s+(?:sendmail|postfix\/(?:local|lmtp|smtpd|smtp|virtual|pipe))\[\d+\]:\s+(.*?):\s+client=(.*)/;
254 $mailid=$id;
255 $mail{$id}{'relay_s'}=$relay_s;
256 debug("For id=$id, found host sender on a 'client' line: $mail{$id}{'relay_s'}");
257 }
258
259 #
260 # See if we received postfix email reject error
261 #
262 elsif (/: reject/) {
263 $MailType||='postfix';
264 # Example:
265 # postfix ?.? : Jan 01 12:00:00 halley postfix/smtpd[9245]: reject: RCPT from unknown[203.156.32.33]: 554 <userx@yahoo.com>: Recipient address rejected: Relay access denied; from=<sender@aol.com> to=<userx@yahoo.com>
266 # postfix 2.1+: Jan 01 12:00:00 localhost postfix/smtpd[11120]: NOQUEUE: reject: RCPT from unknown[62.205.124.145]: 450 Client host rejected: cannot find your hostname, [62.205.124.145]; from=<sender@msn.com> to=<usery@yahoo.com> proto=ESMTP helo=<xxx.com>
267 # postfix ?.? : Jan 01 12:00:00 apollon postfix/smtpd[26553]: 1954F3B8A4: reject: RCPT from unknown[80.245.33.2]: 450 <usery@yahoo.com>: User unknown in local recipient table; from=<sender@msn.com> to=<usery@yahoo.com> proto=ESMTP helo=<xxx.com>
268 my ($mon,$day,$time,$id,$code,$from,$to)=m/(\w+)\s+(\d+)\s+(\d+:\d+:\d+)\s+[\w\-\.\@]+\s+(?:postfix\/(?:local|lmtp|smtpd|smtp|virtual|pipe))\[\d+\]:\s+(.*?):\s+(.*)\s+from=([^\s,]*)\s+to=([^\s,]*)/;
269 # postfix: Jan 01 14:10:16 juni postfix/smtpd[2568]: C34ED1432B: reject: RCPT from relay2.tp2rc.edu.tw[163.28.32.177]: 450 <linda@trieger.org>: User unknown in local recipient table; from=<> proto=ESMTP helo=<rmail.nccu.edu.tw>
270 if (! $mon) { ($mon,$day,$time,$id,$code,$from)=m/(\w+)\s+(\d+)\s+(\d+:\d+:\d+)\s+[\w\-\.\@]+\s+(?:postfix\/(?:local|lmtp|smtpd|smtp|virtual|pipe))\[\d+\]:\s+(.*?):\s+(.*)\s+from=([^\s,]*)/; }
271 $mailid=($id eq 'reject' || $id eq 'NOQUEUE'?'999':$id); # id not provided in log, we take '999'
272 if ($mailid) {
273 # $code='reject: RCPT from unknown[203.156.32.33]: 554 <userx@yahoo.com>: Recipient address rejected: Relay access denied;'
274 rizwank 1.1 # or 'reject: RCPT from unknown[62.205.124.145]: 450 Client host rejected: cannot find your hostname, [62.205.124.145]; from=<sender@msn.com> to=<usery@yahoo.com> proto=ESMTP helo=<xxx.com>'
275 # or 'reject: RCPT from unknown[80.245.33.2]: 450 <usery@yahoo.com>: User unknown in local recipient table;'
276 if ($code =~ /\s+(\d\d\d)\s+/) { $mail{$mailid}{'code'}=$1; }
277 else { $mail{$mailid}{'code'}=999; } # Unkown error
278 if (! $mail{$mailid}{'relay_s'} && $code =~ /from\s+([^\s]+)\s+/) {
279 $mail{$mailid}{'relay_s'}=&trim($1);
280 }
281 $mail{$mailid}{'from'}=&trim($from);
282 if ($to) {
283 $mail{$mailid}{'to'}=&trim($to);
284 }
285 elsif ($code =~ /<(.*)>/) {
286 $mail{$mailid}{'to'}=&trim($1);
287 }
288 $mail{$mailid}{'year'}=$year; ### <CJK>###
289 $mail{$mailid}{'mon'}=$mon;
290 $mail{$mailid}{'day'}=$day;
291 $mail{$mailid}{'time'}=$time;
292 if (! defined($mail{$mailid}{'size'})) { $mail{$mailid}{'size'}='?'; }
293 debug("For id=$mailid, found a postfix error incoming message: code=$mail{$mailid}{'code'} from=$mail{$mailid}{'from'} to=$mail{$mailid}{'to'} time=$mail{$mailid}{'time'}");
294 }
295 rizwank 1.1 }
296 #
297 # See if we received postfix email bounced error
298 #
299 elsif (/stat(us)?=bounced/) {
300 $MailType||='postfix';
301 # Example:
302 # postfix: Sep 9 18:24:23 halley postfix/local[22003]: 12C6413EC9: to=<etavidian@partenor.com>, relay=local, delay=0, status=bounced (unknown user: "etavidian")
303 my ($mon,$day,$time,$id,$to,$relay_r)=m/(\w+)\s+(\d+)\s+(\d+:\d+:\d+)\s+[\w\-\.\@]+\s+(?:postfix\/(?:local|lmtp|smtpd|smtp|virtual|pipe))\[\d+\]:\s+(.*?):\s+to=([^\s,]*)[\s,]+relay=([^\s,]*)/;
304 $mailid=($id eq 'reject'?'999':$id); # id not provided in log, we take '999'
305 if ($mailid) {
306 $mail{$mailid}{'code'}=999; # Unkown error (bounced)
307 $mail{$mailid}{'to'}=&trim($to);
308 $mail{$mailid}{'relay_r'}=&trim($relay_r);
309 $mail{$mailid}{'year'}=$year; ### <CJK>###
310 $mail{$mailid}{'mon'}=$mon;
311 $mail{$mailid}{'day'}=$day;
312 $mail{$mailid}{'time'}=$time;
313 if (! defined($mail{$mailid}{'size'})) { $mail{$mailid}{'size'}='?'; }
314 debug("For id=$mailid, found a postfix bounced incoming message: code=$mail{$mailid}{'code'} to=$mail{$mailid}{'to'} relay_r=$mail{$mailid}{'relay_r'}");
315 }
316 rizwank 1.1 }
317 #
318 # See if we received sendmail reject error
319 #
320 elsif (/, reject/) {
321 $MailType||='sendmail';
322 # Example:
323 # sm-mta: Jul 27 04:06:05 androneda sm-mta[6641]: h6RB44tg006641: ruleset=check_mail, arg1=<7ms93d4ms@topprodsource.com>, relay=crelay1.easydns.com [216.220.57.222], reject=451 4.1.8 Domain of sender address 7ms93d4ms@topprodsource.com does not resolve
324 # sm-mta: Jul 27 06:21:24 androneda sm-mta[11461]: h6RDLNtg011461: ruleset=check_rcpt, arg1=<nobody@nova.dice.net>, relay=freedom.myhostdns.com [66.246.77.42], reject=550 5.7.1 <nobody@nova.dice.net>... Relaying denied
325 # sendmail: Sep 30 04:21:32 halley sendmail[3161]: g8U2LVi03161: ruleset=check_rcpt, arg1=<amber3624@netzero.net>, relay=moon.partenor.fr [10.0.0.254], reject=550 5.7.1 <amber3624@netzero.net>... Relaying denied
326
327 # sendmail: Jan 10 07:37:48 smtp sendmail[32440]: ruleset=check_relay, arg1=[211.228.26.114], arg2=211.228.26.114, relay=[211.228.26.114], reject=554 5.7.1 Rejected 211.228.26.114 found in dnsbl.sorbs.net
328 # sendmail: Jan 10 07:37:08 smtp sendmail[32439]: ruleset=check_relay, arg1=235.Red-213-97-175.pooles.rima-tde.net, arg2=213.97.175.235, relay=235.Red-213-97-175.pooles.rima-tde.net [213.97.175.235], reject=550 5.7.1 Mail from 213.97.175.235 refused. Rejected for bad WHOIS info on IP of your SMTP server - see http://www.rfc-ignorant.org/
329 # sendmail: Jan 10 17:15:42 smtp sendmail[12770]: ruleset=check_relay, arg1=[63.218.84.21], arg2=63.218.84.21, relay=[63.218.84.21], reject=553 5.3.0 Rejected - see http://spamhaus.org/
330 my ($mon,$day,$time,$id,$ruleset,$arg,$relay_s,$code)=m/(\w+)\s+(\d+)\s+(\d+:\d+:\d+)\s+[\w\-\.\@]+\s+(?:sendmail|sm-mta)\[\d+\][:\s]*(.*?):\sruleset=(\w+),\s+arg1=(.*),\s+relay=(.*),\s+(reject=.*)/;
331 # sendmail: Jan 10 18:00:34 smtp sendmail[5759]: i04Axx2c005759: Milter: data, reject=511 Virus found in email!
332 if (! $mon) { ($mon,$day,$time,$id,$ruleset,$code)=m/(\w+)\s+(\d+)\s+(\d+:\d+:\d+)\s+[\w\-\.\@]+\s+(?:sendmail|sm-mta)\[\d+\]:\s+(.*?):\s\w+:\s(\w+),\s+(reject=.*)/; }
333 $mailid=(! $id && $mon?'999':$id); # id not provided in log, we take '999'
334 if ($mailid) {
335 if ($ruleset eq 'check_mail') { $mail{$mailid}{'from'}=$arg; }
336 if ($ruleset eq 'check_rcpt') { $mail{$mailid}{'to'}=$arg; }
337 rizwank 1.1 if ($ruleset eq 'check_relay') { }
338 if ($ruleset eq 'data') { }
339 $mail{$mailid}{'relay_s'}=$relay_s;
340 # $code='reject=550 5.7.1 <amber3624@netzero.net>... Relaying denied'
341 if ($code =~ /=(\d\d\d)\s+/) { $mail{$mailid}{'code'}=$1; }
342 else { $mail{$mailid}{'code'}=999; } # Unkown error
343 $mail{$mailid}{'year'}=$year; ### <CJK>###
344 $mail{$mailid}{'mon'}=$mon;
345 $mail{$mailid}{'day'}=$day;
346 $mail{$mailid}{'time'}=$time;
347 if (! defined($mail{$mailid}{'size'})) { $mail{$mailid}{'size'}='?'; }
348 debug("For id=$mailid, found a sendmail error incoming message: code=$mail{$mailid}{'code'} from=$mail{$mailid}{'from'} to=$mail{$mailid}{'to'} relay_s=$mail{$mailid}{'relay_s'}");
349 }
350 }
351
352 #
353 # See if we send a sendmail (with ctladdr tag) email
354 #
355 elsif (/, ctladdr=/) {
356 $MailType||='sendmail';
357 #
358 rizwank 1.1 # Matched outgoing sendmail/postfix message
359 #
360 my ($mon,$day,$time,$id,$to,$fromorto)=m/(\w+)\s+(\d+)\s+(\d+:\d+:\d+)\s+[\w\-\.\@]+\s+(?:sm-mta|sendmail(?:-out|)|postfix\/(?:local|lmtp|smtpd|smtp|virtual|pipe))\[.*?\]:\s+([^:]*):\s+to=(.*?)[,\s]+ctladdr=([^\,\s]*)/;
361 $mailid=$id;
362 if (m/\s+relay=([^\s,]*)[\s,]/) { $mail{$id}{'relay_r'}=$1; }
363 elsif (m/\s+mailer=local/) { $mail{$id}{'relay_r'}='localhost'; }
364 if (/, stat\=Sent/) { $mail{$id}{'code'}=1; }
365 elsif (/, stat\=User\s+unknown/) { $mail{$id}{'code'}=550; }
366 elsif (/, stat\=Local\s+configuration/) { $mail{$id}{'code'}=451; }
367 elsif (/, stat\=Deferred:\s+(\d*)/) { $mail{$id}{'code'}=$1; }
368 else { $mail{$id}{'code'}=999; }
369 $mail{$mailid}{'year'}=$year; ### <CJK>###
370 $mail{$id}{'mon'}=$mon;
371 $mail{$id}{'day'}=$day;
372 $mail{$id}{'time'}=$time;
373 if (&trim($to)=~/^|/) {
374 # In particular case of mails are sent to a pipe, the ctladdr contains the to
375 $mail{$id}{'to'}=&trim($fromorto);
376 } else {
377 # In most cases
378 $mail{$id}{'to'}=&trim($to);
379 rizwank 1.1 $mail{$id}{'from'}=&trim($fromorto);
380 }
381 if (! defined($mail{$id}{'size'})) { $mail{$id}{'size'}='?'; }
382 debug("For id=$id, found a sendmail outgoing message: to=$mail{$id}{'to'} from=$mail{$id}{'from'} size=$mail{$id}{'size'} relay_r=".($mail{$id}{'relay_r'}||''));
383 }
384
385 #
386 # Matched incoming qmail message
387 #
388 elsif (/info msg .* from/) {
389 # Example: Sep 14 09:58:09 gandalf qmail: 1063526289.292776 info msg 270182: bytes 10712 from <john@john.do> qp 54945 uid 82
390 $MailType||='qmail';
391 #my ($id,$size,$from)=m/info msg \d+: bytes (\d+) from <(.*)>/;
392 my ($id,$size,$from)=m/info msg (\d+): bytes (\d+) from <(.*)>/;
393 $mailid=$id;
394 delete $mail{$mailid}; # If 'info msg' found, we start a new mail. This is to protect from wrong file
395 if (! $mail{$id}{'from'} || $mail{$id}{'from'} ne '<>') { $mail{$id}{'from'}=$from; } # TODO ???
396 $mail{$id}{'size'}=$size;
397 if (m/\s+relay=([^\,]+)[\s\,]/ || m/\s+relay=([^\s\,]+)$/) { $mail{$id}{'relay_s'}=$1; }
398 debug("For id=$id, found a qmail 'info msg' message: from=$mail{$id}{'from'} size=$mail{$id}{'size'}");
399 }
400 rizwank 1.1
401 #
402 # Matched incoming sendmail or postfix message
403 #
404 elsif (/: from=/) {
405 # sm-mta: Jul 28 06:55:13 androneda sm-mta[28877]: h6SDtCtg028877: from=<xxx@mysite.net>, size=2556, class=0, nrcpts=1, msgid=<w1$kqj-9-o2m45@0h2i38.4.m0.5u>, proto=ESMTP, daemon=MTA, relay=smtp.easydns.com [205.210.42.50]
406 # postfix: Jul 3 15:32:26 apollon postfix/qmgr[13860]: 08FB63B8A4: from=<nobody@ns3744.ovh.net>, size=3302, nrcpt=1 (queue active)
407 # postfix: Sep 24 14:45:15 wideboy postfix/qmgr[22331]: 7E0E6196: from=<xxx@hotmail.com>, size=1141 (queue active)
408 my ($id,$from,$size)=m/\w+\s+\d+\s+\d+:\d+:\d+\s+[\w\-\.\@]+\s+(?:sm-mta|sendmail(?:-in|)|postfix\/qmgr|postfix\/nqmgr)\[\d+\]:\s+(.*?):\s+from=(.*?),\s+size=(\d+)/;
409 $mailid=$id;
410 if (! $mail{$id}{'code'}) { $mail{$id}{'code'}=1; } # If not already defined, we define it
411 if (! $mail{$id}{'from'} || $mail{$id}{'from'} ne '<>') { $mail{$id}{'from'}=$from; }
412 $mail{$id}{'size'}=$size;
413 if (m/\s+relay=([^\,]+)[\s\,]/ || m/\s+relay=([^\s\,]+)$/) { $mail{$id}{'relay_s'}=$1; }
414 debug("For id=$id, found a sendmail/postfix incoming message: from=$mail{$id}{'from'} size=$mail{$id}{'size'} relay_s=".($mail{$id}{'relay_s'}||''));
415 }
416
417 #
418 # Matched exchange message
419 #
420 elsif (/^([^\t]+)\t([^\t]+)\t[^\t]+\t([^\t]+)\t([^\t]+)\t([^\t]+)\t[^\t]+\t([^\t]+)\t([^\t]+)\t([^\t]+)\t[^\t]+\t[^\t]+\t([^\t]+)\t[^\t]+\t[^\t]+\t[^\t]+\t[^\t]+\t[^\t]+\t([^\t]+)\t([^\t]+)/) {
421 rizwank 1.1 # date hour GMT ip_s relay_s partner relay_r ip_r to code id size subject from
422 # Example: 2003-8-12 0:58:14 GMT 66.218.66.69 n14.grp.scd.yahoo.com - PACKRAT 192.168.1.2 christina@pirnie.org 1019 bh9e3f+5qvo@eGroups.com 0 0 4281 1 2003-8-12 0:58:14 GMT 0 Version: 6.0.3790.0 - [SRESafeHaven] Re: More Baby Stuff jtluvs2cq@wmconnect.com -
423 $MailType||='exchange';
424 my $date=$1;
425 my $time=$2;
426 my $relay_s=$3;
427 my $partner=$4;
428 my $relay_r=$5;
429 my $to=$6; $to =~ s/\s/%20/g;
430 my $code=$7;
431 my $id=$8;
432 my $size=$9;
433 my $subject=&trim($10);
434 my $from=$11; $from =~ s/\s/%20/g;
435 $id=sprintf("%s_%s_%s",$id,$from,$to);
436 # Check if record is significant record
437 my $ok=0;
438
439 # Code 1031=SMTP End Outbound Transfer
440 if ($code == 1031) { # This is for external bound mails
441 $ok=1;
442 rizwank 1.1 my $savrelay_s=$relay_s;
443 $relay_s=$relay_r; $relay_r=$savrelay_s;
444 #$relay_s=$relay_r;
445 #$relay_r=$partner;
446 $code=1;
447 }
448 # Code 1028=SMTP Store Driver: Message Delivered Locally to Store
449 if ($code == 1028) { # This is for local bound mails
450 $code=1;
451 $ok=1;
452 }
453 # Code 1030=SMTP: Non-Delivered Report (NDR) Generated
454 if ($code == 1030) { # This is for errors.
455 $code=999;
456 $ok=1;
457 }
458
459 if ($ok && !$mail{$id}{'code'} ) {
460 $mailid=$id;
461 if ($date =~ /(\d+)-(\d+)-(\d+)/) {
462 $mail{$id}{'year'}=sprintf("%02s",$1);
463 rizwank 1.1 $mail{$id}{'mon'}=sprintf("%02s",$2);
464 $mail{$id}{'day'}=sprintf("%02s",$3);
465 }
466 if ($time =~ /^(\d+):(\d+):(\d+)/) {
467 $mail{$id}{'time'}=sprintf("%02s:%02s:%02s",$1,$2,$3);
468 }
469 if ( $from eq '<>' && $subject =~ /^Delivery\s+Status/) {
470 $from='postmaster@localhost';
471 }
472 $mail{$id}{'from'}=$from;
473 $mail{$id}{'to'}=$to;
474 $mail{$id}{'code'}=$code;
475 $mail{$id}{'size'}=$size;
476 $mail{$id}{'relay_s'}=$relay_s;
477 $mail{$id}{'relay_r'}=$relay_r;
478 debug("For id=$id, found an exchange message: year=$mail{$id}{'year'} mon=$mail{$id}{'mon'} day=$mail{$id}{'day'} time=$mail{$id}{'time'} from=$mail{$id}{'from'} to=$mail{$id}{'to'} size=$mail{$id}{'size'} code=$mail{$id}{'code'} relay_s=$mail{$id}{'relay_s'} relay_r=$mail{$id}{'relay_r'}");
479 }
480 }
481
482 #
483 # Matched sendmail or postfix "to" message
484 rizwank 1.1 #
485 elsif (/: to=.*stat(us)?=sent/i) {
486 # Example:
487 # postfix: Jan 01 07:27:38 apollon postfix/local[1689]: 2BC793B8A4: to=<jo@jo.com>, orig_to=<webmaster@toto.com>, relay=local, delay=6, status=sent ("|/usr/bin/procmail")
488 my ($mon,$day,$time,$id,$to)=m/(\w+)\s+(\d+)\s+(\d+:\d+:\d+)\s+[\w\-\.\@]+\s+(?:sm-mta|sendmail(?:-out|)|postfix\/(?:local|lmtp|smtpd|smtp|virtual|pipe))\[.*?\]:\s+(.*?):\s+to=(.*?),/;
489 $mailid=$id;
490 $mail{$id}{'code'}='1';
491 if (m/\s+relay=([^\s,]*)[\s,]/) { $mail{$id}{'relay_r'}=$1; }
492 elsif (m/\s+mailer=local/) { $mail{$id}{'relay_r'}='localhost'; }
493 if (m/forwarded as/) {
494 # If 'forwarded as idnewmail' is found, we discard this mail to avoid counting it twice
495 debug("For id=$id, mail was forwarded to other id, we discard it");
496 delete $mail{$id};
497 }
498 else {
499 if (m/\s+orig_to=([^\s,]*)[\s,]/) {
500 # If we have a orig_to, we used it as receiver
501 $mail{$id}{'to'}=&trim($1);
502 $mail{$id}{'forwardedto'}=&trim($to);
503 }
504 else {
505 rizwank 1.1 $mail{$id}{'to'}=&trim($to);
506 }
507 $mail{$mailid}{'year'}=$year; ### <CJK>###
508 $mail{$id}{'mon'}=$mon;
509 $mail{$id}{'day'}=$day;
510 $mail{$id}{'time'}=$time;
511 debug("For id=$id, found a sendmail/postfix record: mon=$mail{$id}{'mon'} day=$mail{$id}{'day'} time=$mail{$id}{'time'} to=$mail{$id}{'to'} relay_r=$mail{$id}{'relay_r'}");
512 }
513 }
514
515 #
516 # Matched qmail "to" record
517 #
518 elsif (/starting delivery/) {
519 # Example: Sep 14 09:58:09 gandalf qmail: 1063526289.574100 starting delivery 251: msg 270182 to local spamreport@john.do
520 # Example: 2003-09-27 11:22:07.039237500 starting delivery 3714: msg 163844 to local name_also_removed@maildomain.com
521 $MailType||='qmail';
522 my ($yea,$mon,$day,$time,$delivery,$id,$relay_r,$to)=();
523 ($mon,$day,$time,$delivery,$id,$relay_r,$to)=m/^(\w+)\s+(\d+)\s+(\d+:\d+:\d+)\s+.*\s+\d+(?:\.\d+)?\s+starting delivery (\d+):\s+msg\s+(\d+)\s+to\s+(.*)?\s+(.*)$/;
524 if (! $id) { ($yea,$mon,$day,$time,$delivery,$id,$relay_r,$to)=m/^(\d+)-(\d+)-(\d+)\s+(\d+:\d+:\d+).*\s+starting delivery (\d+):\s+msg\s+(\d+)\s+to\s+(.*)?\s+(.*)$/; }
525 $mailid=$id;
526 rizwank 1.1 if ($relay_r eq 'local') { $mail{$id}{'relay_r'}='localhost'; }
527 elsif (m/\s+relay=([^\s,]*)[\s,]/) { $mail{$id}{'relay_r'}=$1; }
528 elsif (m/\s+mailer=local/) { $mail{$id}{'relay_r'}='localhost'; }
529 $qmaildelivery{$delivery}=$id; # Save mail id for this delivery to be able to get error code
530 if ($yea) { $mail{$id}{'year'}=$yea; }
531 $mail{$id}{'mon'}=$mon;
532 $mail{$id}{'day'}=$day;
533 $mail{$id}{'time'}=$time;
534 $mail{$id}{'to'}{$delivery}=&trim($to);
535 debug("For id=$id, found a qmail 'start delivery' record: year=".($mail{$id}{'year'}||'')." mon=$mail{$id}{'mon'} day=$mail{$id}{'day'} time=$mail{$id}{'time'} to=$mail{$id}{'to'}{$delivery} relay_r=".($mail{$id}{'relay_r'}||'')." delivery=$delivery");
536 }
537
538 #
539 # Matched qmail status code record
540 #
541 elsif (/delivery (\d+): (\w+):/) {
542 # Example: Sep 14 09:58:09 gandalf qmail: 1063526289.744259 delivery 251: success: did_0+0+1/
543 # Example: 2003-09-27 11:22:07.070367500 delivery 3714: success: did_1+0+0/
544 $MailType||='qmail';
545 my ($delivery,$code)=($1,$2);
546 my $id=$qmaildelivery{$delivery};
547 rizwank 1.1 $mailid=$id;
548 if ($code =~ /success/i) { $mail{$id}{'code'}{$delivery}=1; }
549 elsif ($code =~ /deferral/i) { $mail{$id}{'code'}{$delivery}=999; }
550 else { $mail{$id}{'code'}{$delivery}=999; }
551 debug("For id=$qmaildelivery{$delivery}, found a qmail 'delivery' record: delivery=$delivery code=$mail{$id}{'code'}{$delivery}");
552 }
553 #
554 # Matched qmail end of mail record
555 #
556 elsif (/end msg (\d+)/ && scalar %{$mail{$1}{'to'}}) { # If records for mail id are finished and still mails with no delivery status
557 # Example: Sep 14 09:58:12 gandalf qmail: 1063526292.782444 end msg 270182
558 $MailType||='qmail';
559 my ($id)=($1);
560 $mailid=$id;
561 foreach my $delivery (keys %{$mail{$mailid}{'to'}}) { $mail{$id}{'code'}{$delivery}||=1; }
562 debug("For id=$id, found a qmail 'end msg' record. This replace 'delivery' record for delivery=".join(',',keys %{$mail{$id}{'code'}}));
563 }
564 #
565 # Matched MDaemon log file record
566 #
567 elsif (/^\"(\d\d\d\d)-(\d\d)-(\d\d) (\d\d:\d\d:\d\d)\",\"[^\"]*\",(\w+),\d+,\"([^\"]*)\",\"([^\"]*)\",\"([^\"]*)\",\"[^\"]*\",\"([^\"]*)\",\"([^\"]*)\",\"([^\"]*)\",([\.\d]+),(\d+),(\d+)/) {
568 rizwank 1.1 # Example: "2003-11-06 00:00:42","2003-11-06 00:00:45",SMTPI,9443,"dillon_fm@aaaaa.net","cpeltier@domain.com","","","10.0.0.16","","",0,4563,1
569 $MailType||='mdaemon';
570 my ($id)=($numrecord);
571 if ($5 eq 'SMTPI' || $5 eq 'SMTPO') {
572 $mail{$id}{'year'}=$1;
573 $mail{$id}{'mon'}=$2;
574 $mail{$id}{'day'}=$3;
575 $mail{$id}{'time'}=$4;
576 $mail{$id}{'direction'}=($5 eq 'SMTPI'?'in':'out');
577 $mail{$id}{'from'}=$6;
578 $mail{$id}{'to'}=$7||$8;
579 if ($5 eq 'SMTPI') {
580 $mail{$id}{'relay_s'}=$9;
581 $mail{$id}{'relay_r'}='-';
582 }
583 if ($5 eq 'SMTPO') {
584 $mail{$id}{'relay_s'}=$9;
585 $mail{$id}{'relay_r'}='-';
586 }
587 $mail{$id}{'code'}=1;
588 $mail{$id}{'size'}=$13;
589 rizwank 1.1 $mail{$id}{'extinfo'}="?virus=$10&rbl=$11&heuristicspam=$12&ssl=$14";
590 $mail{$id}{'extinfo'}=~s/\s/_/g;
591 $mailid=$id;
592 }
593 }
594
595
596 #
597 # Write record if all required data were found
598 #
599 if ($mailid) {
600 my $code; my $to;
601 my $delivery=0;
602 my $canoutput=0;
603
604 debug("ID:$mailid RELAY_S:".($mail{$mailid}{'relay_s'}||'')." RELAY_R:".($mail{$mailid}{'relay_r'}||'')." FROM:".($mail{$mailid}{'from'}||'')." TO:".($mail{$mailid}{'to'}||'')." CODE:".($mail{$mailid}{'code'}||''));
605
606 # Check if we can output a mail line
607 if ($MailType eq 'qmail') {
608 if ($mail{$mailid}{'code'} && scalar %{$mail{$mailid}{'code'}}) {
609 # This is a hash variable
610 rizwank 1.1 foreach my $key (keys %{$mail{$mailid}{'code'}}) {
611 $delivery=$key;
612 $code=$mail{$mailid}{'code'}{$key};
613 $to=$mail{$mailid}{'to'}{$key};
614 }
615 $canoutput=1;
616 }
617 }
618 elsif ($MailType eq 'mdaemon') {
619 $code=$mail{$mailid}{'code'};
620 $to=$mail{$mailid}{'to'};
621 $canoutput=1;
622 }
623 else {
624 $code=$mail{$mailid}{'code'};
625 $to=$mail{$mailid}{'to'};
626 if ($mail{$mailid}{'from'} && $mail{$mailid}{'to'}) { $canoutput=1; }
627 if ($mail{$mailid}{'from'} && $mail{$mailid}{'code'} > 1) { $canoutput=1; }
628 if ($mailid && $mail{$mailid}{'code'} > 1) { $canoutput=1; }
629 }
630
631 rizwank 1.1 # If we can
632 if ($canoutput) {
633 &OutputRecord($mail{$mailid}{'year'}?$mail{$mailid}{'year'}:$year,$mail{$mailid}{'mon'},$mail{$mailid}{'day'},$mail{$mailid}{'time'},$mail{$mailid}{'from'},$to,$mail{$mailid}{'relay_s'},$mail{$mailid}{'relay_r'},$code,$mail{$mailid}{'size'},$mail{$mailid}{'forwardto'},$mail{$mailid}{'extinfo'});
634 # Delete mail with generic unknown id (This id can by used by another mail)
635 if ($mailid eq '999') {
636 debug(" Delete mail for id=$mailid",3);
637 delete $mail{$mailid};
638 }
639 # Delete delivery instance for id if qmail (qmail can use same id for several mails with multiple delivery)
640 elsif ($MailType eq 'qmail') {
641 debug(" Delete delivery instances for mail id=$mailid and delivery id=$delivery",3);
642 if ($delivery) {
643 delete $mail{$mailid}{'to'}{$delivery};
644 delete $mail{$mailid}{'code'}{$delivery};
645 }
646 }
647
648 # We flush %mail if too large
649 if (scalar keys %mail > $NBOFENTRYFOFLUSH) {
650 debug("We reach $NBOFENTRYFOFLUSH records in %mail, so we flush mail hash array");
651 #foreach my $id (keys %mail) {
652 rizwank 1.1 # debug(" Delete mail for id=$id",3);
653 # delete $mail{$id};
654 #}
655 %mail=();
656 %qmaildelivery=();
657 }
658
659 }
660 }
661 else {
662 debug("Not interesting row");
663 }
664
665 }
666
667 #foreach my $key (keys %mail) {
668 # print ".$key.$mail{$key}{'to'}.\n";
669 #}
670
671 0;
|
Return to
maillogconvert.pl
CVS log
Up to [RizwankCVS] / geekymedia_web / awstats-6.3 / tools