# Copyright (c) 2010 Flowerfire, Inc. All Rights Reserved. postfix = { plugin_version = "1.15" # 2006-08-09 - GMF - 1.1beta - Added tracking of SpamAssassin information, when present. # This was created to attempt to fix a number of problems with the old format, # without changing the existing format, as that would take too long and may not succeed. # A 'ground up' approach has been undertaken. This has now been updated to v7 Salang parse # filter and renamed to the current label as the first choice for this log (the old postfix has # been removed). # 2007-06-12 - gas - 1.2beta - added support for a variant (Postfix: 2.4.0) # 2007-09-13 - 1.2 - KBB - renumbered per new beta policy and changed name from beta_postfix.cfg # 2007-10-25 - 1.3 - KBB - fixed bug where from and to not being captured on blocked messages # 2007-10-26 - 1.4 - gas - added support for relay-ip containing port info # (relay=mail.domain.com[123.21.123.21]:25) # 2008-01-28 - 1.5 - MSG for GMF - In 'Handle to= lines', moved 'v.message = $2;' above second regexp # match line. Added support for letters other than A-F in message ID. # 2008-02-07 - 1.6 - KBB - Renamed message_id field to queue_id and added code to put the value # of message-id (message-id=<748735E9.0203024@nowhere.com>) field message_id. Some code clean-up. # Set v.status at top to fix bug. Use set_collected for message-id to mid mapping to save memory. # 2008-04-10 - 1.7 - GMF - Added support for Amavis lines. # 2008-04-10 - 1.8 - GMF - Added support for MailScanner lines. # 2009-04-07 - 1.9 - GMF - Added filters to suppress message_id and queue_id, and switched to rand_hash # 2010-09-09 - 1.10 - GMF - Added detection of Brightmail variant # 2010-09-09 - 1.11 - GMF - Added support for random info at the end of the Client host section # above in reject lines. # 2010-09-17 - 1.12 - GMF - Changed the label to specify "Brightmail Gateway" instead of just "Brightmail" # 2011-01-25 - 1.13 - GAS - Added support for sasl_method and sasl_user # 2011-07-28 - 1.14 - KBB - All messages_ count fields are not cleared after each accept with true. Clearing # all of them is not always necessary, but because the Amavis section won't accept if there's a key, and # because future sections may not accept, it is the most flexible approach. Also corrected the number of # backslashes in some regular expressions in the parsing filter and fixed problem with 1.11 accept. # 2011-08-15 - 1.15 - KBB - Added support for Subject: lines and subject= lines and added subject field. # log file format info, latest changes info.1.manufacturer = "Postfix" info.1.device = "mail server" info.1.version.1 = "2.4.0" info.2.manufacturer = "Symantec" info.2.device = "Brightmail Gateway" info.2.version.1 = "" # The name of the log format log.format.format_label = "Postfix or Brightmail Gateway Log Format" log.miscellaneous.log_data_type = "syslog_required" log.miscellaneous.log_format_type = "mail_server" # The log is in this format if there is a match this regular expression log.format.autodetect_regular_expression = "postfix|outbound-mta/[a-z]+[[][0-9]+[]]" # Search this many lines for the regex for auto-detect (this is set so high because of some variants # that produce many lines at the top of the file that do not contain our test "postfix", # for performance, this could be reduced without much danger). log.format.autodetect_lines = "100" # All log field parsing will be done using the parsing filters log.format.parse_only_with_filters = "true" # This discards uncollected entries after 10000 lines log.format.collected_entry_lifespan = "10000" log.format.discard_expired_entries = true # Log fields log.fields = { from = { type = "hierarchical" hierarchy_dividers = "@" left_to_right = false leading_divider = false } to = { type = "hierarchical" hierarchy_dividers = "@" left_to_right = false leading_divider = false } subject = "" size = "" user = "" rbl_list = "" amavis_result = "" mailscanner_result = "" client_hostname = "" client_ip.type = "host" relay_hostname = "" relay_ip = "" direction = "" status = "" sc_status = "" # nrcpt = "" # counter = "" queue_id = "" message_id = { itemnums_hash_function = "rand_sum" } messages_delivered = "" messages_processed = "" messages_scanned = "" messages_blocked = "" messages_expired = "" messages_delivered = "" messages_bounced = "" bytes_delivered = "" bytes_processed = "" bytes_blocked = "" bytes_expired = "" bytes_bounced = "" spam_detected = "" sasl_method = "" sasl_username = "" } # log.fields # log.filter_initialization = ` #node message_id_info; #v.message_id_infos = ''; #node message_id_infos = 'v.message_id_infos'; #` # Log Parsing Filters #Jan 31 00:01:43 inbound.postfix.gwy postfix/qmgr[27189]: [ID 197553 mail.info] B32B0182A: from=, size=2378, nrcpt=2 (queue active) #Jan 31 00:01:43 inbound.postfix.gwy postfix/smtp[10117]: [ID 197553 mail.info] B32B0182A: to=, relay=192.168.22.22[192.168.22.22]:25, delay=0.96, delays=0.85/0.01/0/0.1, dsn=2.0.0, status=sent (250 2.0.0 Ok: queued as 82CF7181A0) #Jan 31 00:01:43 inbound.postfix.gwy postfix/smtp[10117]: [ID 197553 mail.info] B32B0182A: to=, relay=192.168.22.22[192.168.22.22]:25, delay=0.96, delays=0.85/0.01/0/0.1, dsn=2.0.0, status=sent (250 2.0.0 Ok: queued as 82CF7181A0) log.parsing_filters.parse = ` v.status = ""; if (matches_regular_expression(v.syslog_message, '^[^ ]+\\\\[*[^]]*\\\\]*:? ([0-9A-Z]+|NOQUEUE|spamd): (.*)$')) then ( v.key = $1; v.message = $2; set_collected_field(v.key, 'date', get_collected_field('', 'date')); set_collected_field(v.key, 'time', get_collected_field('', 'time')); set_collected_field(v.key, 'logging_device', get_collected_field('', 'logging_device')); set_collected_field(v.key, 'syslog_priority', get_collected_field('', 'syslog_priority')); if (v.key ne 'NOQUEUE') then set_collected_field(v.key, 'queue_id', v.key); if (matches_regular_expression(v.message, '^client=([^[]*)\\\\[([^]]*)\\\\]$')) then ( if ($1 ne 'localhost') then ( set_collected_field(v.key, 'client_hostname', $1); set_collected_field(v.key, 'client_ip', $2); ); ); if (matches_regular_expression(v.message, '^client=([^[]*)\\\\[([^]]*)\\\\], sasl_method=([^,]+), sasl_username=([^$]+)$')) then ( if ($1 ne 'localhost') then ( set_collected_field(v.key, 'client_hostname', $1); set_collected_field(v.key, 'client_ip', $2); set_collected_field(v.key, 'sasl_method', $3); set_collected_field(v.key, 'sasl_username', $4); ); ); # Handle blocked messages else if (matches_regular_expression(v.message, '^(reject): RCPT from ([^[]*)\\\\[([^]]*)\\\\]: ([0-9][0-9][0-9]) [^;]+;(.*)$')) then ( set_collected_field(v.key, 'status', $1); v.client_hostname = $3; set_collected_field(v.key, 'client_hostname', v.client_hostname); set_collected_field(v.key, 'client_ip', $3); set_collected_field(v.key, 'sc_status', $4); v.message = $5; if (matches_regular_expression(v.message, '^ *Client host \\\\[[0-9.]+\\\\] blocked using ([^;]+);[^;]*;* (.*)$')) then ( set_collected_field(v.key, 'rbl_list', $1); v.message = $2; ); # 2010-09-09 - 1.11 - GMF - Added support for random info at the end of the Client host section # above in reject lines. if (matches_regular_expression(v.message, ' *from=<([^>]*)> to=<([^>]*)> ')) then ( set_collected_field(v.key, 'from', $1); set_collected_field(v.key, 'to', $2); ); set_collected_field(v.key, 'messages_blocked', 1); set_collected_field(v.key, 'bytes_blocked', get_collected_field(v.key, 'size')); accept_collected_entry(v.key, true); # Make sure each of these is only associated with one line set_collected_field(v.key, 'messages_blocked', 0); set_collected_field(v.key, 'messages_scanned', 0); set_collected_field(v.key, 'messages_expired', 0); set_collected_field(v.key, 'messages_bounced', 0); set_collected_field(v.key, 'messages_delivered', 0); set_collected_field(v.key, 'messages_processed', 0); ); # Handle from= lines else if (matches_regular_expression(v.message, '^from=([^,]*),(.*)$')) then ( v.from = $1; v.message = $2; if (matches_regular_expression(v.from, '^<([^>]*)>$')) then v.from = $1; set_collected_field(v.key, 'from', v.from); if (matches_regular_expression(v.message, '^ size=([0-9]*)(.*)$')) then ( set_collected_field(v.key, 'size', $1); ) else if (matches_regular_expression(v.message, '^ status=([^,]*),')) then ( v.status = $1; set_collected_field(v.key, 'status', $1); ); if (v.status eq 'expired') then ( set_collected_field(v.key, 'messages_expired', 1); set_collected_field(v.key, 'bytes_expired', get_collected_field(v.key, 'size')); ); set_collected_field(v.key, 'messages_processed', 1); set_collected_field(v.key, 'bytes_processed', get_collected_field(v.key, 'size')); accept_collected_entry(v.key, true); # Make sure each of these is only associated with one line set_collected_field(v.key, 'messages_blocked', 0); set_collected_field(v.key, 'messages_scanned', 0); set_collected_field(v.key, 'messages_expired', 0); set_collected_field(v.key, 'messages_bounced', 0); set_collected_field(v.key, 'messages_delivered', 0); set_collected_field(v.key, 'messages_processed', 0); ); # from= # Handle to= lines else if (matches_regular_expression(v.message, '^to=([^,]*),(.*)$')) then ( v.to = $1; v.message = $2; set_collected_field(v.key, 'date', get_collected_field('', 'date')); set_collected_field(v.key, 'time', get_collected_field('', 'time')); set_collected_field(v.key, 'logging_device', get_collected_field('', 'logging_device')); set_collected_field(v.key, 'syslog_priority', get_collected_field('', 'syslog_priority')); if (matches_regular_expression(v.to, '^<([^>]*)>$')) then v.to = $1; set_collected_field(v.key, 'to', v.to); if (matches_regular_expression(v.message, '^ orig_to=([^,]*),(.*)$')) then ( v.message = $2; ); if (matches_regular_expression(v.message, '^ relay=([^,]*),(.*)$')) then ( v.relay_ip = $1; v.message = $2; # relay_ip contains both hostname and IP, strip that out if no port if (matches_regular_expression(v.relay_ip, '^([^[]*)\\\\[([0-9.]*)\\\\]$')) then ( set_collected_field(v.key, 'relay_hostname', $1); v.relay_ip = $2; ); # relay_ip contains both hostname and IP (and sometimes port), strip that out if there is a port else if (matches_regular_expression(v.relay_ip, '^([^[]*)\\\\[([0-9.]*)\\\\]:[0-9]+$')) then ( set_collected_field(v.key, 'relay_hostname', $1); v.relay_ip = $2; ); set_collected_field(v.key, 'relay_ip', v.relay_ip); ); if (matches_regular_expression(v.message, '^ delay=([^,]*),(.*)$')) then ( set_collected_field(v.key, 'delay', $1); v.message = $2; ); # if status=sent if (matches_regular_expression(v.message, ' status=([^, ]*) (.*)$')) then ( v.status = $1; v.message = $2; set_collected_field(v.key, 'status', v.status); if (matches_regular_expression(v.message, '^\\\\(host [^ ]+ said: (.*)\\\\)$')) then v.message = $1; if (matches_regular_expression(v.message, '^\\\\(([0-9]+) ([^)]*)\\\\)')) then ( v.message = $2; set_collected_field(v.key, 'response_code', $2); set_collected_field(v.key, 'response_message', $3); ); ); # if status= # 1.6 - KBB - removed ^ above for same result - why process these formats separately? - # whatever is between status= and the beginning is thrown away here anyway # # 1.2beta - duplicated the if status=sent section above and changed the regex so that ' status=' does not have to start a line - gas # else if (matches_regular_expression(v.message, ' status=([^, ]*) (.*)$')) then ( # v.status = $1; # set_collected_field(v.key, 'status', v.status); # v.message = $2; # if (matches_regular_expression(v.message, '^\\\\(host [^ ]+ said: (.*)$')) then # v.message = $1; # if (matches_regular_expression(v.message, '^\\\\(*([0-9]+) ([^)]*)\\\\)')) then ( # set_collected_field(v.key, 'response_code', $2); # set_collected_field(v.key, 'response_message', $3); # ); # v.message = $2; # ); # 1.2beta - duplicated the if status=sent section above and changed the regex so that ' status=' does not have to start a line - gas if (matches_regular_expression(v.message, 'queued as ([0-9A-Z]+)\\\\)')) then ( set_collected_field(v.key, 'size', get_collected_field($1, 'size')); if (get_collected_field($1, 'client_hostname') eq '(empty)') then set_collected_field(v.key, 'client_hostname', get_collected_field($1, 'client_hostname')); if (get_collected_field($1, 'client_ip') eq '(empty)') then set_collected_field(v.key, 'client_ip', get_collected_field($1, 'client_ip')); ); set_collected_field(v.key, 'messages_processed', 0); set_collected_field(v.key, 'bytes_processed', 0); if (v.status eq 'sent') then ( set_collected_field(v.key, 'messages_delivered', 1); set_collected_field(v.key, 'bytes_delivered', get_collected_field(v.key, 'size')); ); else if ((v.status eq 'bounce') or (v.status eq 'bounced')) then ( set_collected_field(v.key, 'messages_bounced', 1); set_collected_field(v.key, 'bytes_bounced', get_collected_field(v.key, 'size')); ); accept_collected_entry(v.key, true); # Make sure each of these is only associated with one line set_collected_field(v.key, 'messages_blocked', 0); set_collected_field(v.key, 'messages_scanned', 0); set_collected_field(v.key, 'messages_expired', 0); set_collected_field(v.key, 'messages_bounced', 0); set_collected_field(v.key, 'messages_delivered', 0); #set_collected_field(v.key, 'messages_processed', 0); # already reset above ); # if to= # Handle message-id lines else if (matches_regular_expression(v.message, '^message-id=<([^>]+)>')) then ( v.mid = $1; set_collected_field(v.key, 'message_id', v.mid); v.from = get_collected_field(v.key, 'from'); v.to = get_collected_field(v.key, 'to'); #message_id_info = subnode_by_name(message_id_infos, replace_all(replace_all($1, '$', '_'), '.', '_')); #set_subnode_value(message_id_info, "from", replace_all(v.from, '.', '_')); #set_subnode_value(message_id_info, "to", replace_all(v.from, '.', '_')); # If these are empty, they'll be empty anyway if they aren't saved here. if (v.from ne '(empty)') then ( set_collected_field(v.mid, 'from', v.from); # echo("FROM: " . v.from); ); if (v.to ne '(empty)') then ( set_collected_field(v.mid, 'to', v.to); # echo("TO: " . v.to); ); ); # if message-id # Handle "hold" lines else if (matches_regular_expression(v.message, '^hold: .*; from=<([^>]+)> to=<([^>]+)>')) then ( set_collected_field(v.key, 'from', $1); set_collected_field(v.key, 'to', $2); ); # hold # Handle lines with "Subject:" or "subject=" #Jul 24 05:27:48 lamppost postfix/cleanup[20396]: 55D7CD3220: warning: header Subject: Blackberry Curve 8520 so 492 reais. Home Theater por 398 reais e muito mais. Nao Perca. from localhost[127.0.0.1]; from=<5ea492e1368b354dbd648ad6679d031d.User@mouse.snail.com.br> to= proto=ESMTP helo=: POST-LA else if (matches_regular_expression(v.message, 'Subject: (.*) from [^[]*\\\\[[^]]*\\\\]; ')) then ( set_collected_field(v.key, 'subject', $1); ); #Apr 18 03:00:05 lamppost postfix/cleanup[69693]: FB35FCA7AU: subject=jTTU~GU>ZAL~JTULRIR>~AYL>U>KL~UIL>K else if (matches_regular_expression(v.message, 'subject=(.*)$')) then ( set_collected_field(v.key, 'subject', $1); ); # Handle spamd lines else if (matches_regular_expression(v.message, '^result: Y (.*)$')) then ( v.message = $1; if (matches_regular_expression(v.message, 'mid=<([^>]+)>')) then ( v.mid = $1; set_collected_field(v.key, 'message_id', v.mid); #message_id_info = subnode_by_name(message_id_infos, replace_all($1, '.', '_')); #set_collected_field(v.key, 'from', node_value(subnode_by_name(message_id_info, "from"))); #set_collected_field(v.key, 'to', node_value(subnode_by_name(message_id_info, "to"))); set_collected_field(v.key, 'from', get_collected_field(v.mid, 'from')); set_collected_field(v.key, 'to', get_collected_field(v.mid, 'to')); ); set_collected_field(v.key, 'spam_detected', 1); accept_collected_entry(v.key, true); # Make sure each of these is only associated with one line set_collected_field(v.key, 'messages_blocked', 0); set_collected_field(v.key, 'spam_detected', 0); set_collected_field(v.key, 'messages_scanned', 0); set_collected_field(v.key, 'messages_expired', 0); set_collected_field(v.key, 'messages_bounced', 0); set_collected_field(v.key, 'messages_delivered', 0); set_collected_field(v.key, 'messages_processed', 0); ); ); # if v.key and message # Handle amavis lines # e.g. amavis[16921]: (16121-09) Passed CLEAN, [12.34.56.78] [98.76.54.32] -> , Message-ID: <001a01c8303c$abe0ebce0$52d5494b@Someone>, mail_id: 3xnJA6gLw2HS, Hits: 1.441, queued_as: CA123C1234B, 1251 ms # e.g. Nov 26 09:26:44 mail amavis[20582]: (20582-06) Blocked SPAM, [24.222.0.30] [216.167.248.203] -> , Message-ID: <012201c8303d$60feaaa0$cbf8a7d8@NAQSQ>, mail_id: Ra+--ULDuZbi, Hits: 6.586, 1179 ms if (matches_regular_expression(v.syslog_message, '^amavis[[][0-9]+[]]: [(][0-9-]+[)] ([^ ]+) ([^,]+), [[]([0-9.]+)[]] [[]([0-9.]+)[]] <([^>]*)> -> <([^>]*)>, Message-ID: <([^>]*)>, mail_id: ([^ ]+), Hits: ([^,]+), (.*)')) then ( v.remainder = $10; v.amavis_result = $1; v.amavis_reason = $2; v.source_ip = $3; v.from = $5; v.to = $6; v.message_id = $7; # If there is a queued_as line, we can use that as the key, and we don't need to accept yet. # If there is no queued_as line, we need to set and accept this as a single entry. if (matches_regular_expression(v.remainder, 'queued_as: ([^,]+),')) then ( v.key = $1; ); else ( v.key = ""; set_collected_field(v.key, 'source_ip', v.source_ip); set_collected_field(v.key, 'from', v.from); set_collected_field(v.key, 'to', v.to); set_collected_field(v.key, 'message_id', v.message_id); ); set_collected_field(v.key, 'amavis_result', v.amavis_result . ' ' . v.amavis_reason); if ((v.amavis_result eq "Blocked") and (v.amavis_reason eq "SPAM")) then set_collected_field(v.key, 'spam_detected', 1); if (v.amavis_result eq "Blocked") then set_collected_field(v.key, 'messages_blocked', 1); set_collected_field(v.key, 'messages_scanned', 1); # If key is "", that means there was no queue ID above, so we need to accept this as a separate entry. if (v.key eq "") then ( accept_collected_entry(v.key, false); ); ); # if amavis # Handle MailScanner lines # e.g. MailScanner[17221]: Content Checks: Detected and have disarmed web bug, phishing tags in HTML message in BAB62C78093.94DA3 from bounce-12345-987654@abc.def.com else if (matches_regular_expression(v.syslog_message, '^MailScanner[[][0-9]+[]]: (.*)')) then ( v.mailscanner_message = $1; # Ignore Requeue lines if (matches_regular_expression(v.mailscanner_message, '^Requeue: ')) then ( ); # Ignore New Batch lines else if (matches_regular_expression(v.mailscanner_message, '^New Batch: ')) then ( ); # Ignore "Virus and Content Scanning: Starting" lines else if (matches_regular_expression(v.mailscanner_message, '^Virus and Content Scanning: Starting')) then ( ); else ( set_collected_field('', 'mailscanner_result', v.mailscanner_message); set_collected_field('', 'messages_scanned', 1); accept_collected_entry('', true); # Make sure each of these is only associated with one line set_collected_field(v.key, 'messages_blocked', 0); set_collected_field(v.key, 'messages_scanned', 0); set_collected_field(v.key, 'messages_expired', 0); set_collected_field(v.key, 'messages_bounced', 0); set_collected_field(v.key, 'messages_delivered', 0); set_collected_field(v.key, 'messages_processed', 0); ); ); # if mailscanner ` # parse log.filters = { suppress_message_id = { label = "$lang_admin.log_filters.suppress_message_id_label" comment = "$lang_admin.log_filters.suppress_message_id_comment" value = `message_id = "[omitted]"` } # suppress_message_id suppress_queue_id = { label = "$lang_admin.log_filters.suppress_queue_id_label" comment = "$lang_admin.log_filters.suppress_queue_id_comment" value = `queue_id = "[omitted]"` } # suppress_queue_id } # log.filters # Database fields database.fields = { from = { itemnums_hash_function = "rand_sum" } to = { itemnums_hash_function = "rand_sum" } subject = "" client_hostname = "" client_ip = "" relay_hostname = "" relay_ip = "" status = "" sc_status = "" rbl_list = "" amavis_result = "" mailscanner_result = "" queue_id = { itemnums_hash_function = "rand_sum" } message_id = { itemnums_hash_function = "rand_sum" } sasl_method = "" sasl_username = "" } # database.fields database.numerical_fields = { messages_delivered = { default = true } messages_processed = { default = true } messages_blocked = "" messages_scanned = "" messages_expired = "" messages_bounced = "" bytes_delivered = { type = "int" integer_bits = 64 display_format_type = "bandwidth" default = true } bytes_processed = { type = "int" integer_bits = 64 display_format_type = "bandwidth" default = true } bytes_blocked = { type = "int" integer_bits = 64 display_format_type = "bandwidth" } bytes_expired = { type = "int" integer_bits = 64 display_format_type = "bandwidth" } bytes_bounced = { type = "int" integer_bits = 64 display_format_type = "bandwidth" } spam_detected = "" # nrcpt = { ## label = "$lang_stats.field_labels.nrcpt" # default = false # requires_log_field = false # type = "int" # display_format_type = "integer" # entries_field = true # } # nrcpt # # size = { # label = "$lang_stats.field_labels.size" # default = false # requires_log_field = true # log_field = "size" # type = "int" # integer_bits = 64 # display_format_type = "bandwidth" # } # size } # database.numerical_fields create_profile_wizard_options = { # How the reports should be grouped in the report menu report_groups = { date_time_group = "" } # report_groups } # create_profile_wizard_options } # postfix