# Copyright (c) 2010 Flowerfire, Inc. All Rights Reserved. iron_port = { plugin_version = "1.4.9" info.1.manfacturer = "IronPort" info.1.device = "C Series Secure Email" info.1.version.1 = "AsyncOS 4.7" # 4.7.0-148 info.1.version.2 = "AsyncOS 4.6" # 4.6.1-031 info.2.manfacturer = "Cisco/IronPort" info.2.device = "C Series Secure Email" info.2.version.1 = "AsyncOS 4.7" # 4.7.0-148 info.2.version.2 = "AsyncOS 4.6" # 4.6.1-031 # 2006-08-28 - GMF - 1.1beta - Fixed "no v.icid defined" # 2006-09-14 - GMF - 1.2beta - Fixed a bug where an entry could get incorrect ICID values, # if the ICID was reused. # 2006-12-18 - GMF - 1.2beta - Added support for UNIX syslog style header; added support # for multiple MID in "message done" # 2007-01-01 - GMF - 1.4beta - converted to syslog_required # 2007-03-07 - GMF - 1.5beta - Fixed bug where Info: would get chopped off by logger chopper. # - Restored extraction of date/time when available. # 2007-04-27 - KBB - 1.5.1beta - Fixed bug where aborted entries were not being accepted. # 2007-05-14 - KBB - 1.6beta - Added support for Async OS version 4.6 which does not have Info: # and Warning: before lines, at least with syslog. # - Added rekeying to avoid losing syslog info. # - Added handling of rewritten MIDs to pick up more antivirus info. # 2007-08-05 - GMF - 1.7beta - Changed collected_entry_lifespan to 1000, to fix memory leak. # 2007-09-11 - KBB - 1.7 - Renumbered per new beta policy. # 2008-02-27 - GMF - 1.8 - Added CASE spam line tracking, and quarantine tracking; switched to accepting on queued, # to get better AV info. # 2008-03-17 - GMF - 1.9 - Fixed CASE to properly categorize under new antispam_result field. # 2008-04-09 - GMF - 1.10 - Added tracking of VOF quarantine lines. # 2008-04-11 - GMF - 1.11 - Fixed tracking of SRBS rejections. # 2008-06-20 - GMF - 1.12 - Enhanced tracking of aborted messages, to show recipients, and count message deliveries aborted. # 2008-09-01 - GMF - 1.2 - Added support for Delayed HAT REJECT sessions. # 2008-09-02 - gas - 1.2.1 - Fixed tracking of "reverse_dns_host" where "interface" contains a space # 2008-09-05 - GMF - 1.3 - Improved action field to track all actions; enhanced Actions report to include pie chart. # 2008-09-10 - GMF - 1.3.1 - Improved reporting of quarantined messages. # 2008-09-10 - GMF - 1.4 - Added messages_spam_positive and messages_virus_positive fields # 2008-11-23 - GMF - 1.4.1 - Relaxed the advanced SBRS regexp to allow non-numerical scores, and non-uppercase lists # 2009-01-01 - GMF - 1.4.2 - Removed pie chart from Actions, because it was causing an error in v8 (incompatible final_step) # 2009-01-28 - GMF - 1.4.3 - Added support for optional IP address in HAT REJECT lines # 2009-12-07 - GMF - 1.4.4 - Considered RELAY as a non-rejected action # 2010-04-05 - GMF - 1.4.5 - Added support for double-quoted subjects # 2010-04-10 - GMF - 1.4.6 - Fixed handling of lines without an XYZ: prefix # 2010-10-01 - 1.4.7 - MSG - Edited info lines. # 2010-11-19 - SAA - 1.4.8 - Added "location" database field and changed address log field to address.type="host" to enable Countries/Regions/Cities Reports from GeoIP. # 2010-01-25 - GMF - 1.4.9 - Fixed issues with tracking and reporting of bounced messages. # The name of the log format log.format.format_label = "IronPort C-Series Log Format" log.miscellaneous.log_data_type = "syslog_required" log.miscellaneous.log_format_type = "mail_server" # The log is in this format if any of the first ten lines match this regular expression log.format.autodetect_expression = ` matches_regular_expression(volatile.log_data_line, "Info: Version: [^ ]+ SN: [0-9A-Z-]+$") or matches_regular_expression(volatile.log_data_line, "Info: Bounced: .*From:.*To:") or matches_regular_expression(volatile.log_data_line, "New SMTP ICID") or matches_regular_expression(volatile.log_data_line, "MID [0-9]+ ICID [0-9]+ From: ") or matches_regular_expression(volatile.log_data_line, "Info: Begin Logfile") ` # The format of dates and times in this log # log.format.date_format = "mmm dd hh:mm:ss yyyy" # log.format.time_format = "mmm dd hh:mm:ss yyyy" # All log field parsing will be done using the parsing filters log.format.parse_only_with_filters = "true" log.format.collected_entry_lifespan = 100000 # Log fields log.fields = { # date = "" # time = "" type = "" from = { type = "hierarchical" hierarchy_dividers = "@" left_to_right = false leading_divider = "false" } # from to = { type = "hierarchical" hierarchy_dividers = "@" left_to_right = false leading_divider = "false" } # to sbrs_action = "" sbrs_score = { type = "hierarchical" hierarchy_dividers = "." left_to_right = true leading_divider = "false" } # sbrs_score sbrs_list = "" message_id = "" subject = "" antispam_result = "" antivirus_result = "" bytes_transferred = "" interface = "" interface_host = "" address.type = "host" reverse_dns_host = "" response = "" action = "" mid = "" icid = "" rid = "" reason = "" response = "" warning_message = "" events = "" messages_queued = "" messages_delivered = "" messages_delayed = "" messages_bounced = "" messages_quarantined = "" } # log.fields # Declare filter variables log.filter_initialization = ` v.icid = ''; v.mid_to_icid = ''; v.mid_to_recipients = ''; node recipients; node this_recipient; string recipient; int line_number = 1; ` # Log Parsing Filters log.parsing_filters.parse = ` v.message = v.syslog_message; v.date = get_collected_field('', 'date'); v.time = get_collected_field('', 'time'); v.message_type = ""; # 2007-01-01 - GMF - Changed to syslog_required, so removed date extraction - GMF # 2007-03-07 - GMF - Restored this; it's sometimes part of a direct-logged file (non syslog), and must be handled. if (matches_regular_expression(v.message, '^[A-Za-z]* ?([A-Za-z]* *[0-9]* [0-9:]* [0-9]*) (.*)$')) then ( v.date = normalize_date($1, "mmm dd hh:mm:ss yyyy"); v.time = normalize_time($1, "mmm dd hh:mm:ss yyyy"); v.message = $2; ); else if (matches_regular_expression(v.message, '^([A-Z][a-z][a-z] *[0-9]+ [0-9][0-9]:[0-9][0-9]:[0-9][0-9]) [^:]+: (.*)$')) then ( # v.date = normalize_date($1, "mmm dd hh:mm:ss yyyy"); # v.time = normalize_time($1, "mmm dd hh:mm:ss yyyy"); v.message = $2; ); # e.g. Nov 29 16:33:33 171.22.21.140 testmaillog: Info: Begin Logfile # This is UNIX Syslog format; do we need to change this plug-in to Syslog Required? - GMF 2006-12-15 #else if (matches_regular_expression(current_log_line(), '^([A-Z][a-z][a-z] *[0-9]+) ([0-9][0-9]:[0-9][0-9]:[0-9][0-9]) [0-9.]+ [^:]+: (.*)$')) then ( # v.date = normalize_date($1, "mmm dd"); # v.time = $2; # v.message = $3; #); # Chop off the leading log name, if any. E.g., chop off "testmaillog:" in the following line. # 2007-03-07 - GMF - Added check for Info or Warning; otherwise they'll get chopped off here too, and won't match below, e.g.: # Nov 29 16:33:33 174.21.20.140 testmaillog: Info: MID 8205709 ready 3091100 bytes from # 2007-07-30 - GMF - Changed [^ :] to [^:] because the log name can contain a space, e.g.: # Jan 06 00:30:16 210.175.215.21 mail logs: Info: ICID 119060 close # 2010-04-20 - GMF - Changed [^:] to [^:0-9] because otherwise it can consume the whole line up to From: in this case: # MID 93699 ICID 62059 From: if (!matches_regular_expression(v.message, '^(Info|Warning|MID|Bounced):') and matches_regular_expression(v.message, '^[^:0-9]+: (.*)$')) then ( v.message = $1; ); # To support Async OS 4.6, we now need to chop this off too and save it if (matches_regular_expression(v.message, '^(Info|Warning): (.*)$')) then ( v.message_type = $1; v.message = $2; ); if (v.message ne "") then ( ## Handle Info lines #if (matches_regular_expression(v.message, '^Info: (.*)$')) then ( #v.message = $1; # 1.2.1 # Jul 05 03:03:30 blah-ip1 syslog-ng: Info: New SMTP ICID 62123066 interface Data 9 (111.111.111.111) address 111.111.111.111 reverse dns host blah.blah.blah.blah.com verified yes # if (matches_regular_expression(v.message, '^New SMTP ICID ([0-9]+) interface ([^ ]+) \\\\(([^)]*)\\\\) address ([^ ]*) reverse dns host ([^ ]*) ')) then ( if (matches_regular_expression(v.message, '^New SMTP ICID ([0-9]+) interface ([^(]+) \\\\(([^)]*)\\\\) address ([^ ]*) reverse dns host ([^ ]*) ')) then ( v.icid = $1; set_collected_field(v.icid, 'date', v.date); set_collected_field(v.icid, 'time', v.time); set_collected_field(v.icid, 'interface', $2); set_collected_field(v.icid, 'interface_host', $3); set_collected_field(v.icid, 'address', $4); set_collected_field(v.icid, 'reverse_dns_host', $5); ); # 1.2.1 # else if (matches_regular_expression(v.message, '^New SMTP ICID ([0-9]+) interface ([^ ]+) address ([^ ]*)')) then ( else if (matches_regular_expression(v.message, '^New SMTP ICID ([0-9]+) interface ([^a]+) address ([^ ]*)')) then ( v.icid = $1; set_collected_field(v.icid, 'date', v.date); set_collected_field(v.icid, 'time', v.time); set_collected_field(v.icid, 'interface', $2); set_collected_field(v.icid, 'address', $3); ); # 2007-05-15 - KBB - These won't be used due to removal of other sections below. #else if (matches_regular_expression(v.message, '^New SMTP DCID ([0-9]+) interface ([^ ]+) address ([^ ]+) port [0-9]+')) then ( # v.dcid = $1; # set_collected_field(v.dcid, 'date', v.date); # set_collected_field(v.dcid, 'time', v.time); # set_collected_field(v.dcid, 'interface', $2); # set_collected_field(v.dcid, 'address', $3); #); # Parse SBRS lines else if (matches_regular_expression(v.message, '^ICID ([0-9]+) SBRS (.*)$')) then set_collected_field($1, 'sbrs_score', $2); # Parse Delayed HAT REJECT sessions. # e.g.: # Aug 29 00:00:02 mail_logs: Info: ICID 8696 Delayed HAT REJECT continuing session for recipient logging # Aug 29 00:00:02 mail_logs: Info: ICID 8696 Delayed HAT REJECT Message from: spammer@spammer.com # Aug 29 00:00:03 mail_logs: Info: ICID 8696 Delayed HAT REJECT Message recipient: # Aug 29 00:00:03 mail_logs: Info: ICID 8696 close else if (matches_regular_expression(v.message, '^ICID ([0-9]+) Delayed HAT REJECT continuing')) then ( set_collected_field($1, 'date', v.date); set_collected_field($1, 'time', v.time); ); else if (matches_regular_expression(v.message, '^ICID ([0-9]+) Delayed HAT REJECT Message from: (.*)$')) then ( v.icid = $1; v.from = $2; # 2009-01-28 - GMF - Strip off IP addresses in lines like this: # Dec 22 00:00:03 mail_logs: Info: ICID 3200355 Delayed HAT REJECT Message recipient: (12.34.56.78) if (matches_regular_expression(v.from, '^([^ ]+) [(][^)]+[)]$')) then v.from = $1; if (matches_regular_expression(v.from, '^<([^>]+)>$')) then v.from = $1; set_collected_field(v.icid, 'from', v.from); set_collected_field(v.icid, 'action', 'rejected'); set_collected_field(v.icid, 'messages_rejected', 1); ); else if (matches_regular_expression(v.message, '^ICID ([0-9]+) Delayed HAT REJECT Message recipient: +([^ ].*)$')) then ( v.icid = $1; v.from = $2; v.to = $2; # 2009-01-28 - GMF - Strip off IP addresses in lines like this: # Dec 22 00:00:03 mail_logs: Info: ICID 3200355 Delayed HAT REJECT Message from: spammer@there.com (12.34.56.67) if (matches_regular_expression(v.to, '^([^ ]+) [(][^)]+[)]$')) then v.to = $1; if (matches_regular_expression(v.to, '^<([^>]+)>$')) then v.to = $1; set_collected_field(v.icid, 'to', v.to); set_collected_field(v.icid, 'action', 'rejected'); set_collected_field(v.icid, 'messages_rejected', 1); ); else if (matches_regular_expression(v.message, '^ICID ([0-9]+) close') and (get_collected_field($1, 'messages_rejected') == 1)) then ( set_collected_field($1, 'events', 1); accept_collected_entry($1, false); ); # Parse advanced SBRS lines, e.g.: # ICID 43569971 ACCEPT SG None match ALL SBRS -9.2 # ICID 43569968 ACCEPT SG None match ALL SBRS rfc1918 else if (matches_regular_expression(v.message, '^ICID ([0-9]+) ([A-Z]+) [A-Z]+ ([A-Za-z]+) match [^ ]+ SBRS (.*)$')) then ( set_collected_field($1, 'sbrs_action', $2); set_collected_field($1, 'sbrs_list', $3); set_collected_field($1, 'sbrs_score', $4); # If this is an SBRS reject, this is the last we'll see of this message, so accept it now # 2009-12-07 - GMF - Considered RELAY as a non-rejected action. if (($2 ne "ACCEPT") and ($2 ne "RELAY")) then ( set_collected_field($1, 'action', 'rejected'); set_collected_field($1, 'messages_rejected', 1); set_collected_field($1, 'events', 1); accept_collected_entry($1, false); ); ); # if SBRS # Handle rewritten MIDs #2006-08-17T07:23:51+0200 ip01 mail info mail_log_syslog: Info: MID 461156 rewritten to MID 462475 by antivirus #2006-08-16T04:30:23+0200 ip01 mail info mail_log_syslog: Info: MID 458360 rewritten to MID 458361 by alt-rcpt-to-filter filter \'CF_RedirectToEmail-Collector\' else if (matches_regular_expression(v.message, '^MID ([0-9]+) rewritten to MID ([0-9]+) ')) then ( v.old_mid = $1; v.mid = $2; if (get_collected_field(v.mid, 'rekeyed') ne 'rekeyed') then ( rekey_collected_entry(v.old_mid, v.mid); set_collected_field(v.mid, 'rekeyed', 'rekeyed'); ); # We just rekeyed this MID; make sure that the ICID is associated with the new MID #echo("v.mid_to_icid: " . node_as_string('v.mid_to_icid')); if ('v.mid_to_icid'?{v.old_mid}) then @'v.mid_to_icid'{v.mid} = @'v.mid_to_icid'{v.old_mid}; #echo("v.mid_to_icid: " . node_as_string('v.mid_to_icid')); #echo("v.mid_to_recipients: " . node_as_string('v.mid_to_recipients')); if ('v.mid_to_recipients'?{v.old_mid}) then rename_node('v.mid_to_recipients'{v.old_mid}, v.mid); #echo("v.mid_to_recipients: " . node_as_string('v.mid_to_recipients')); ); # if MID rewritten to MID # Get MID->ICID converter else if (matches_regular_expression(v.message, '^Start MID ([0-9]+) ICID ([0-9]+)$')) then ( v.mid = $1; v.icid = $2; if (get_collected_field(v.mid, 'rekeyed') ne 'rekeyed') then ( rekey_collected_entry('', v.mid); set_collected_field(v.mid, 'rekeyed', 'rekeyed'); ); # Remember the MID->ICID conversion set_subnode_value('v.mid_to_icid', v.mid, v.icid); # Copy fields from the ICID to the MID set_collected_field(v.mid, 'date', get_collected_field(v.icid, 'date')); set_collected_field(v.mid, 'time', get_collected_field(v.icid, 'time')); set_collected_field(v.mid, 'interface', get_collected_field(v.icid, 'interface')); set_collected_field(v.mid, 'interface_host', get_collected_field(v.icid, 'interface_host')); set_collected_field(v.mid, 'address', get_collected_field(v.icid, 'address')); set_collected_field(v.mid, 'reverse_dns_host', get_collected_field(v.icid, 'reverse_dns_host')); set_collected_field(v.mid, 'sbrs_action', get_collected_field(v.icid, 'sbrs_action')); set_collected_field(v.mid, 'sbrs_list', get_collected_field(v.icid, 'sbrs_list')); set_collected_field(v.mid, 'sbrs_score', get_collected_field(v.icid, 'sbrs_score')); ); # If Start MID # Handle Delivery start DCID lines # 2006-09-14 - GMF - We can't do this, because by the time we get here, the DCID may have # been re-used, and the sbrs_score, for instance, may be from another connection than the one # corresponding to this MID. The Start MID lines capture all this anyway, at a better time, # don't they? # else if (matches_regular_expression(v.message, '^Delivery start DCID ([0-9]+) MID ([0-9]+) ')) then ( # v.dcid = $1; # v.mid = $2; # set_collected_field(v.mid, 'date', get_collected_field(v.dcid, 'date')); # set_collected_field(v.mid, 'time', get_collected_field(v.dcid, 'time')); # set_collected_field(v.mid, 'interface', get_collected_field(v.dcid, 'interface')); # set_collected_field(v.mid, 'address', get_collected_field(v.dcid, 'address')); # set_collected_field(v.mid, 'sbrs_action', get_collected_field(v.icid, 'sbrs_action')); # set_collected_field(v.mid, 'sbrs_list', get_collected_field(v.icid, 'sbrs_list')); # set_collected_field(v.mid, 'sbrs_score', get_collected_field(v.icid, 'sbrs_score')); # ); else if (matches_regular_expression(v.message, '^MID ([0-9]+) (.*)$')) then ( v.mid = $1; # if (subnode_exists('v.mid_to_icid', v.mid)) then ( # v.icid = node_value(subnode_by_name('v.mid_to_icid', v.mid)); v.message = $2; if (get_collected_field(v.mid, 'rekeyed') ne 'rekeyed') then ( rekey_collected_entry('', v.mid); set_collected_field(v.mid, 'rekeyed', 'rekeyed'); ); # Discard RID information if (matches_regular_expression(v.message, '^RID \\\\[[^]]+\\\\] (.*)$')) then v.message = $1; if (matches_regular_expression(v.message, '^ICID [0-9]+ From: (.*)$')) then ( v.from = $1; if (matches_regular_expression(v.from, '^<([^>]+)>$')) then v.from = $1; set_collected_field(v.mid, 'from', $1); ); else if (matches_regular_expression(v.message, '^ICID [0-9]+ RID ([0-9]+) To: (.*)$')) then ( v.rid = $1; v.to = $2; if (matches_regular_expression(v.to, '^<([^>]+)>$')) then v.to = $1; # Remember this recipient for this MID in v.mid_to_recipients set_subnode_value(subnode_by_name('v.mid_to_recipients', v.mid), v.rid, v.to); set_subnode_value(subnode_by_name('v.mid_to_recipients', v.mid), 'last_access', line_number); # debug_message("Set node 'v.mid_to_recipients." . v.mid . "." . v.rid . " to " . v.to . "\\n"); # set_collected_field(v.icid, 'to', $1); ); else if (matches_regular_expression(v.message, '^Message-ID \\'([^\\']*)\\'')) then set_collected_field(v.mid, 'message_id', $1); else if (matches_regular_expression(v.message, '^Subject \\'([^\\']*)\\'')) then set_collected_field(v.mid, 'subject', $1); else if (matches_regular_expression(v.message, '^Subject "([^"]*)"')) then set_collected_field(v.mid, 'subject', $1); else if (matches_regular_expression(v.message, '^ready ([0-9]+) bytes')) then ( set_collected_field(v.mid, 'bytes_transferred', $1); ); # if ready...bytes # Mark it queued when we see a 'ready N bytes' line, because queued lines only occur in # recent versions of the format, and these lines appear in all versions. # 2008-02-27 - GMF - I'm changing this back to accepting on "queued for delivery" even though it will break processing of old # logs, because the spam information comes *after* the "ready N bytes" line in the log data, and *before* # queued for delivery. So we have to wait for queued for delivery in order to get spam information. # people with older logs will just have to use an older plug-in (from 2008-02-26). # else if (matches_regular_expression(v.message, '^ready ([0-9]+) bytes')) then ( else if (matches_regular_expression(v.message, '^queued for delivery')) then ( if (get_collected_field(v.mid, 'rekeyed') ne 'rekeyed') then ( rekey_collected_entry('', v.mid); set_collected_field(v.mid, 'rekeyed', 'rekeyed'); ); v.icid = ''; if (subnode_exists('v.mid_to_icid', v.mid)) then ( v.icid = node_value(subnode_by_name('v.mid_to_icid', v.mid)); set_subnode_value(subnode_by_name('v.mid_to_icid', v.mid), 'last_access', line_number); ); # Add database entry for sender set_collected_field(v.mid, 'date', v.date); set_collected_field(v.mid, 'time', v.time); set_collected_field(v.mid, 'action', 'queued'); set_collected_field(v.mid, 'messages_queued', 1); set_collected_field(v.mid, 'messages_delivered', 0); set_collected_field(v.mid, 'messages_bounced', 0); set_collected_field(v.mid, 'messages_delayed', 0); set_collected_field(v.mid, 'messages_quarantined', get_collected_field(v.mid, 'quarantined')); set_collected_field(v.mid, 'reason', ''); set_collected_field(v.mid, 'response', ''); set_collected_field(v.mid, 'to', ''); set_collected_field(v.mid, 'action', 'sent'); set_collected_field(v.mid, 'icid', v.icid); set_collected_field(v.mid, 'mid', v.mid); set_collected_field(v.mid, 'rid', ''); set_collected_field(v.mid, 'events', 1); accept_collected_entry(v.mid, true); ); # If "queued for delivery" else if (matches_regular_expression(v.message, "^Response '([^ ]+)")) then ( set_collected_field(v.mid, 'response', $1); ); else if (matches_regular_expression(v.message, '^Brightmail (.*)$')) then ( set_collected_field(v.mid, 'anntispam_result', $1); ); else if (matches_regular_expression(v.message, '^(antivirus .*)$')) then ( if (get_collected_field(v.mid, 'antivirus_result') ne '(empty)') then set_collected_field(v.mid, 'antivirus_result', get_collected_field(v.mid, 'antivirus_result') . '; ' . $1); else set_collected_field(v.mid, 'antivirus_result', $1); if (contains($1, 'positive')) then set_collected_field(v.mid, 'messages_virus_positive', 1); ); # 2008-04-09 - GMF - Handle quarantine lines # e.g. 2007-07-23T12:20:44+0200 mail01 mail info mail_logs-syslog: Info: MID 1381053 quarantined to \"Outbreak\" (VOF rule:OUTBREAK_0001397) else if (matches_regular_expression(v.message, '^(quarantined .*)$')) then ( set_collected_field(v.mid, 'quarantined', 1); if (get_collected_field(v.mid, 'antivirus_result') ne '(empty)') then set_collected_field(v.mid, 'antivirus_result', get_collected_field(v.mid, 'antivirus_result') . '; ' . $1); else set_collected_field(v.mid, 'antivirus_result', $1); ); else if (matches_regular_expression(v.message, '^using engine: (CASE .*)$')) then ( if (get_collected_field(v.mid, 'antispam_result') ne '(empty)') then set_collected_field(v.mid, 'antispam_result', get_collected_field(v.mid, 'antispam_result') . '; ' . $1); else set_collected_field(v.mid, 'antispam_result', $1); if (contains($1, 'positive')) then set_collected_field(v.mid, 'messages_spam_positive', 1); ); ); # if MID # Handle ISQ: Quarantined else if (matches_regular_expression(v.message, '^ISQ: (Quarantined) MID (.*)$')) then ( v.mid = $2; set_collected_field(v.mid, 'quarantined', 1); if (get_collected_field(v.mid, 'antivirus_result') ne '(empty)') then set_collected_field(v.mid, 'antivirus_result', get_collected_field(v.mid, 'antivirus_result') . '; ' . $1); else set_collected_field(v.mid, 'antivirus_result', $1); ); # if quarantined # Handle ISQ: Tagging MID N for quarantine else if (matches_regular_expression(v.message, '^ISQ: Tagging MID ([0-9]+) for quarantine$')) then ( v.mid = $1; v.action = "tagged for quarantine"; set_collected_field(v.mid, 'quarantined', 1); if (get_collected_field(v.mid, 'antivirus_result') ne '(empty)') then set_collected_field(v.mid, 'antivirus_result', get_collected_field(v.mid, 'antivirus_result') . '; ' . v.action); else set_collected_field(v.mid, 'antivirus_result', v.action); ); # if tagged for quarantine # Accept on "Message done DCID 8943260 MID 5621 to RID [0, 1] []" lines else if (matches_regular_expression(v.message, '^Message done DCID ([0-9]+) MID ([0-9]+) to [RID ]*\\\\[([0-9, ]+)\\\\]')) then ( #v.dcid = $1; v.mid = $2; v.rid = $3; if (get_collected_field(v.mid, 'rekeyed') ne 'rekeyed') then ( rekey_collected_entry('', v.mid); set_collected_field(v.mid, 'rekeyed', 'rekeyed'); ); # Get the ICID from the MID v.icid = ''; #echo("mid_to_icid size: " . num_subnodes('v.mid_to_icid')); #echo("mid_to_icid size: " . node_as_string('v.mid_to_icid')); if (subnode_exists('v.mid_to_icid', v.mid)) then ( v.icid = node_value(subnode_by_name('v.mid_to_icid', v.mid)); set_subnode_value(subnode_by_name('v.mid_to_icid', v.mid), 'last_access', line_number); # delete_node(subnode_by_name('v.mid_to_icid', v.mid)); ); # Get the recipient from the MID and RID. #echo("### v.rid: " . v.rid); #echo("mid_to_recipients size: " . num_subnodes('v.mid_to_recipients')); if (subnode_exists('v.mid_to_recipients', v.mid)) then ( recipients = subnode_by_name('v.mid_to_recipients', v.mid); #echo("recipients: " . node_as_string(recipients)); set_subnode_value(subnode_by_name('v.mid_to_recipients', v.mid), 'last_access', line_number); # Loop through all RIDs for this message, adding an event for each. v.remaining_rids = v.rid; while (matches_regular_expression(v.remaining_rids, '^([0-9]+), (.*)$') or matches_regular_expression(v.remaining_rids, '^([0-9]+)()$')) ( v.this_rid = $1; v.remaining_rids = $2; #echo("v.this_rid: " . v.this_rid); #echo("v.remaining_rids: " . v.remaining_rids); recipient = ''; if (subnode_exists(recipients, v.this_rid)) then ( recipient = node_value(subnode_by_name(recipients, v.this_rid)); #echo("FOUND recipient: " . recipient); ); #echo("v.date=" . v.date); #echo("v.time=" . v.time); # Accept the received entry set_collected_field(v.mid, 'date', v.date); set_collected_field(v.mid, 'time', v.time); set_collected_field(v.mid, 'to', recipient); set_collected_field(v.mid, 'action', 'delivered'); set_collected_field(v.mid, 'messages_queued', 0); set_collected_field(v.mid, 'messages_delivered', 1); set_collected_field(v.mid, 'messages_bounced', 0); set_collected_field(v.mid, 'messages_delayed', 0); set_collected_field(v.mid, 'messages_quarantined', get_collected_field(v.mid, 'quarantined')); set_collected_field(v.mid, 'reason', ''); set_collected_field(v.mid, 'response', ''); set_collected_field(v.mid, 'action', 'delivered'); set_collected_field(v.mid, 'icid', v.icid); set_collected_field(v.mid, 'mid', v.mid); set_collected_field(v.mid, 'rid', v.this_rid); set_collected_field(v.mid, 'events', 1); accept_collected_entry(v.mid, true); ); # while rids # delete_node(recipients); ); # if mid exists ); # if Message Done # Handle "Message finished MID 5621 done" # We now handle this in "queued for delivery" lines # Accept a corrupt entry here to clear this MID out of memory else if (matches_regular_expression(v.message, '^Message finished MID ([0-9]+) done')) then ( v.mid = $1; set_collected_field(v.mid, 'date', '{corrupt}'); set_collected_field(v.mid, 'time', '{corrupt}'); accept_collected_entry(v.mid, false); # accept_collected_entry(v.mid, true); ); # v.icid = ''; # if (subnode_exists('v.mid_to_icid', v.mid)) then # v.icid = node_value(subnode_by_name('v.mid_to_icid', $1)); # # # Add database entry for sender # set_collected_field(v.mid, 'date', v.date); # set_collected_field(v.mid, 'time', v.time); # set_collected_field(v.mid, 'messages_queued', 1); # set_collected_field(v.mid, 'messages_delivered', 0); # set_collected_field(v.mid, 'messages_bounced', 0); # set_collected_field(v.mid, 'messages_delayed', 0); # set_collected_field(v.mid, 'reason', ''); # set_collected_field(v.mid, 'response', ''); # set_collected_field(v.mid, 'to', ''); # set_collected_field(v.mid, 'action', 'sent'); # set_collected_field(v.mid, 'icid', v.icid); # set_collected_field(v.mid, 'mid', v.mid); # set_collected_field(v.mid, 'rid', ''); # accept_collected_entry(v.mid, false); # # ); # If Message finished # Handle Bounced or Delayed lines, # e.g., Tue Sep 27 12:16:42 2005 Info: Bounced: DCID 8943502 MID 5615 From: To: RID 419 - 5.1.0 - Unknown address error ('550', ['5.7.1 SPF check failed: 63.36.202.241 is not authorized to send in the name of "somewhere.com".']) Headers: ['^*PARTS: 1', '^FNAME: Sue', '^LNAME: Smith', '^SID: 84403', '^CODE: law4:0wpq2:pvdh3:KHQ--HHkjGMWSLLLhd7Jp', '^VIEW: hj2uqtpwqz:p11wap2:k1ppff3:radmrrvxv1', '^IPADDR: 121.205.186.14', '^WCMID: 31654', '^*TO: bob@school.edu'] else if (matches_regular_expression(v.message, '^(Bounced|Delayed): (.*)$')) then ( v.bounced_or_delayed = $1; v.message = $2; v.mid = ''; v.icid = ''; v.rid = ''; # Discard M:N information from the front if (matches_regular_expression(v.message, '^[0-9]+:[0-9]+ (.*)$')) then ( v.message = $1; ); if (matches_regular_expression(v.message, '^DCID:* ([0-9]+) (.*)$')) then ( v.message = $2; ); if (matches_regular_expression(v.message, '^MID ([0-9]+) (.*)$')) then ( v.mid = $1; v.message = $2; ); if (matches_regular_expression(v.message, '^Message ([0-9]+) to ([0-9]+) - (.*)$')) then ( v.mid = $1; v.rid = $2; v.message = $3; ); if (get_collected_field(v.mid, 'rekeyed') ne 'rekeyed') then ( rekey_collected_entry('', v.mid); set_collected_field(v.mid, 'rekeyed', 'rekeyed'); ); if (matches_regular_expression(v.message, '^From:<([^>]+)> (.*)$')) then ( set_collected_field(v.mid, 'from', $1); v.message = $2; ); if (matches_regular_expression(v.message, '^To:<([^>]+)> (.*)$')) then ( set_collected_field(v.mid, 'to', $1); v.message = $2; ); # Extract RID after To: if (matches_regular_expression(v.message, '^RID ([0-9]+) - (.*)$')) then ( v.rid = $1; v.message = $2; ); # Strip off the headers if (contains(v.message, ' Headers:')) then ( v.headers_index = index(v.message, ' Headers:'); v.message = substr(v.message, 0, v.headers_index); ); if (matches_regular_expression(v.message, '^to RID ([0-9]+) - (.*)$')) then ( v.rid = $1; v.message = $2; ); # 2010-01-25 - GMF - 1.4.9 - Fixed issues with tracking and reporting of bounced messages. if (matches_regular_expression(v.message, '^Bounced by destination server with response: (.*)$')) then ( v.message = $1; ); # Make sure we don't carry over previous reasons set_collected_field(v.mid, 'reason', ''); set_collected_field(v.mid, 'response', ''); # Extract the response message from new format lines by looking for the quote # following the [. This can be a " or a '. Find the closing one to get the message. if (matches_regular_expression(v.message, "^([0-9a-z.]+ - [^(]+) \\\\('([0-9]+)', \\\\[(.*)$")) then ( set_collected_field(v.mid, 'reason', $1); v.response_code = $2; v.message = $3; v.response_quote = substr(v.message, 0, 1); v.message = substr(v.message, 1); v.end_index_response = index(v.message, v.response_quote); v.response = ''; if (v.end_index_response != -1) then v.response = substr(v.message, 0, v.end_index_response); set_collected_field(v.mid, 'response', v.response_code . ': ' . v.response); ); # Extract the reason/response from old-format lines if (matches_regular_expression(v.message, '^Reason: "([^"]+)" Response: "([^"]+)"')) then ( set_collected_field(v.mid, 'reason', $1); set_collected_field(v.mid, 'response', $2); ); # Look up ICID from MID if we don't have it if ((v.icid eq '') and (v.mid ne '') and subnode_exists('v.mid_to_icid', v.mid)) then ( v.icid = node_value(subnode_by_name('v.mid_to_icid', v.mid)); set_subnode_value(subnode_by_name('v.mid_to_icid', v.mid), 'last_access', line_number); # delete_node(subnode_by_name('v.mid_to_icid', v.mid)); ); # Get the recipient from the MID and RID recipient = ''; if (subnode_exists('v.mid_to_recipients', v.mid)) then ( recipients = subnode_by_name('v.mid_to_recipients', v.mid); set_subnode_value(subnode_by_name('v.mid_to_recipients', v.mid), 'last_access', line_number); if (subnode_exists(recipients, v.rid)) then ( recipient = node_value(subnode_by_name(recipients, v.rid)); set_collected_field(v.mid, 'to', recipient); ); # if valid RID ); # if valid ICID # Accept entry set_collected_field(v.mid, 'date', v.date); set_collected_field(v.mid, 'time', v.time); set_collected_field(v.mid, 'messages_queued', 0); set_collected_field(v.mid, 'messages_delivered', 0); set_collected_field(v.mid, 'brightmail_result', ''); set_collected_field(v.mid, 'antivirus_result', ''); set_collected_field(v.mid, 'antispam_result', ''); set_collected_field(v.mid, 'messages_quarantined', get_collected_field(v.mid, 'quarantined')); if (v.bounced_or_delayed eq 'Bounced') then ( set_collected_field(v.mid, 'action', 'bounced'); set_collected_field(v.mid, 'messages_bounced', 1); set_collected_field(v.mid, 'messages_delayed', 0); ); else ( set_collected_field(v.mid, 'action', 'delayed'); set_collected_field(v.mid, 'messages_bounced', 0); set_collected_field(v.mid, 'messages_delayed', 1); ); set_collected_field(v.mid, 'icid', v.icid); set_collected_field(v.mid, 'mid', v.mid); set_collected_field(v.mid, 'rid', v.rid); set_collected_field(v.mid, 'events', 1); accept_collected_entry(v.mid, true); ); # if Bounced # Handle "Message aborted" lines else if (matches_regular_expression(v.message, '^Message aborted MID ([0-9]+)')) then ( v.mid = $1; v.icid = ''; if (subnode_exists('v.mid_to_icid', v.mid)) then ( v.icid = node_value(subnode_by_name('v.mid_to_icid', v.mid)); set_subnode_value(subnode_by_name('v.mid_to_icid', v.mid), 'last_access', line_number); # delete_node(subnode_by_name('v.mid_to_icid', v.mid)); ); if (get_collected_field(v.mid, 'rekeyed') ne 'rekeyed') then ( rekey_collected_entry('', v.mid); set_collected_field(v.mid, 'rekeyed', 'rekeyed'); ); set_collected_field(v.mid, 'date', v.date); set_collected_field(v.mid, 'time', v.time); set_collected_field(v.mid, 'action', 'aborted'); set_collected_field(v.mid, 'messages_queued', 0); set_collected_field(v.mid, 'messages_delivered', 0); set_collected_field(v.mid, 'messages_bounced', '0'); set_collected_field(v.mid, 'messages_delayed', 0); set_collected_field(v.mid, 'messages_quarantined', get_collected_field(v.mid, 'quarantined')); set_collected_field(v.mid, 'reason', ''); set_collected_field(v.mid, 'response', ''); set_collected_field(v.mid, 'messages_aborted', '1'); set_collected_field(v.mid, 'icid', v.icid); set_collected_field(v.mid, 'mid', v.mid); set_collected_field(v.mid, 'events', 1); accept_collected_entry(v.mid, true); if (subnode_exists('v.mid_to_recipients', v.mid)) then ( recipients = subnode_by_name('v.mid_to_recipients', v.mid); #echo("recipients: " . node_as_string(recipients)); # Add an entry for each recipient, showing it was a message delivery rejected foreach this_recipient recipients ( if (node_name(this_recipient) ne "last_access") then ( v.this_rid = node_name(this_recipient); recipient = node_value(this_recipient); # Accept the received entry set_collected_field(v.mid, 'date', v.date); set_collected_field(v.mid, 'time', v.time); set_collected_field(v.mid, 'to', recipient); set_collected_field(v.mid, 'messages_queued', 0); set_collected_field(v.mid, 'messages_delivered', 0); set_collected_field(v.mid, 'messages_bounced', 0); set_collected_field(v.mid, 'messages_delayed', 0); set_collected_field(v.mid, 'messages_aborted', 0); set_collected_field(v.mid, 'message_deliveries_aborted', 1); set_collected_field(v.mid, 'messages_quarantined', get_collected_field(v.mid, 'quarantined')); set_collected_field(v.mid, 'reason', ''); set_collected_field(v.mid, 'response', ''); set_collected_field(v.mid, 'action', 'delivery_aborted'); set_collected_field(v.mid, 'icid', v.icid); set_collected_field(v.mid, 'mid', v.mid); set_collected_field(v.mid, 'rid', v.this_rid); set_collected_field(v.mid, 'events', 1); accept_collected_entry(v.mid, true); ); # if not last_access ); # foreach recipient ); # if v.mid_to_recipients subnode exists ); # if aborted #); # if Info # Handle Warning: lines # Note that in version 4.6 there are no Warning/Info labels, but there are also # so many types of messages that are not matched above that storing them all seems # pointless. Testing for likely and actual patterns from warning messages is a # compromise, though some might be missed. #else if (matches_regular_expression(v.message, '^Warning: (.*)$')) then ( if ((v.message_type eq "Warning") or (contains(get_collected_field('', 'syslog_priority'), 'warning')) or # Only some syslogs have this ((v.message_type eq "") and (matches_regular_expression(v.message, '([iI]nvalid|[pP]roblem|[wW]arning|could not be reached)')))) then ( set_collected_field('', 'date', v.date); set_collected_field('', 'time', v.time); set_collected_field('', 'action', 'warning'); set_collected_field('', 'warnings', '1'); set_collected_field('', 'warning_message', v.message); set_collected_field('', 'events', 1); accept_collected_entry('', false); ); ); # if (v.message ne "") # Every 10000 lines, clear out anything from memory that hasn't been used for the past 10000 lines. #echo("line_number: " . line_number); line_number++; if (line_number % 10000 == 0) then ( # echo("line_number: " . line_number); # echo("v.mid_to_icid size: " . num_subnodes('v.mid_to_icid')); # echo("v.mid_to_recipients size: " . num_subnodes('v.mid_to_recipients')); # Delete old data from mid_to_icid node n; int i = 0; for (int i = num_subnodes('v.mid_to_icid') - 1; i >= 0; i--) ( n = subnode_by_number('v.mid_to_icid', i); if (line_number - node_value(subnode_by_name(n, 'last_access')) > 10000) then ( # echo("Deleting from mid_to_icid: " . node_as_string(n)); delete_node(n); ); ); # foreach n # Delete old data from mid_to_recipients node n; int i = 0; for (int i = num_subnodes('v.mid_to_recipients') - 1; i >= 0; i--) ( n = subnode_by_number('v.mid_to_recipients', i); if (line_number - node_value(subnode_by_name(n, 'last_access')) > 10000) then ( # echo("Deleting from mid_to_recipients: " . node_as_string(n)); delete_node(n); ); ); # foreach n # Clear internal.delete_nodes_from_changes; otherwise, our deletions will each create a new node there. delete_node('internal.delete_nodes_from_changes'); ); # if %10000 ` # Database fields database.fields = { date_time = "" day_of_week = "" hour_of_day = "" action = "" from = "" to = "" sbrs_action = "" sbrs_list = "" sbrs_score = "" message_id = "" subject = "" antispam_result = "" antivirus_result = "" interface = "" interface_host = "" address = "" reverse_dns_host = "" response = "" reason = "" response = "" icid = "" mid = "" rid = "" warning_message = "" location = "" } # database.fields database.numerical_fields = { events = { label = "$lang_stats.field_labels.events" default = true requires_log_field = false entries_field = true } # events messages_delivered = { label = "$lang_stats.field_labels.messages_delivered" default = true requires_log_field = false } # messages_delivered messages_queued = { label = "$lang_stats.field_labels.messages_queued" default = true requires_log_field = false } # messages_queued messages_rejected = { default = false requires_log_field = false } # messages_rejected messages_aborted = { default = false requires_log_field = false } # messages_aborted messages_spam_positive = { default = false requires_log_field = false } # messages_spam_positive messages_virus_positive = { default = false requires_log_field = false } # messages_virus_positive message_deliveries_aborted = { default = false requires_log_field = false } # message_deliveries_aborted messages_quarantined = { label = "$lang_stats.field_labels.messages_quarantined" default = true requires_log_field = false } # messages_quarantined # 2010-01-25 - GMF - 1.4.9 - Fixed issues with tracking and reporting of bounced messages. messages_bounced = { default = true requires_log_field = false } # messages_bounced messages_delayed = { label = "$lang_stats.field_labels.messages_delayed" default = true requires_log_field = false } # messages_delayed bytes_transferred = { label = "$lang_stats.field_labels.bytes_transferred" default = false requires_log_field = true log_field = "bytes_transferred" type = "float" display_format_type = "bandwidth" } # bytes_transferred warnings = { label = "$lang_stats.field_labels.warnings" default = false requires_log_field = false } # warnings } # database.numerical_fields log.filters = { } # log.filters create_profile_wizard_options = { ######### START OF FINAL_STEP CODE final_step = ` include "templates.admin.profiles.setup_reports_util"; string profile = "profiles." . volatile.new_profile_name; # Start with the standard reports based on remaining DB fields add_standard_reports(profile); ` ######## END OF FINAL_STEP CODE # How the reports should be grouped in the report menu report_groups = { date_time_group = "" from = true to = true sbrs_action = true sbrs_list = true sbrs_score = true message_id = true subject = true antispam_result = true antivirus_result = true interface = true interface_host = true address = true reverse_dns_host = true response = true action = true reason = true response = true warning_message = true icid = true mid = true rid = true } # report_groups } # create_profile_wizard_options } # iron_port