# Copyright (c) 2010 Flowerfire, Inc. All Rights Reserved. communigate_pro = { plugin_version = "1.3" info.1.manufacturer = "CommuniGate Systems" info.1.device = "CommuniGate Pro" info.1.version.1 = "" info.1.version.2 = "5.1" info.1.version.3 = "5.2" # 2007-09-11 - 1.0 - KBB - added version number and changed file name from beta_communigate_pro.cfg # 2008-03-24 - 1.1 - MSG - extracted source_host for SMTPI lines # 2008-06-03 - 1.2 - GMF - Added tracking of IMAP messages # 2009-09-10 - 1.3 - KBB - Added support for version 5.2. Improved processing of relayed and # delivered messages by accepting with true because there may be multiple recipients and # by eliminating duplication of ACCOUNT/delivered and DEQUEUER/LOCAL/delivered. Added basic support # for DEQUEUER/LIST/relayed. Eliminated underused operation field and added an action field just # for delivery type. Added Queue ID field. # The name of the log format log.format.format_label = "CommuniGate Pro Log Format" log.miscellaneous.log_data_type = "firewall" log.miscellaneous.log_format_type = "mail_server" # The log is in this format if any of the first ten lines match this regular expression log.format.autodetect_regular_expression = "^[0-9][0-9]:[0-9][0-9]:[0-9][0-9]\\.[0-9][0-9]+ [0-9]* [A-Z0-9]*-[0-9]" # The format of dates and times in this log log.format.time_format = "hh:mm:ss" log.format.date_format = "yyyy-mm-dd" # Extract the date from the filename # log.format.global_date_filename_regular_expression = "([0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9])" # All log field parsing will be done using the parsing filters log.format.parse_only_with_filters = "true" # Log fields log.fields = { date = "" time = "" source_host = "" queue_id = "" pop_account = "" imap_account = "" # source_address = "" local_account = "" # only a log field, not in database # operation = "" action = "" protocol = "" sender = { type = "hierarchical" hierarchy_dividers = "@" left_to_right = false leading_divider = false } # sender recipient = { type = "hierarchical" hierarchy_dividers = "@" left_to_right = false leading_divider = false } # recipient subject = "" messages_sent = "" bytes_sent = "" pop_logins = "" pop_messages_retrieved = "" pop_messages_deleted = "" pop_bytes_retrieved = "" imap_logins = "" } # log.fields log.filter_initialization = ` v.previous_pipe_key = ''; v.date = ''; v.path = ''; ` # Log Parsing Filters log.parsing_filters.parse = ` if ((v.path ne current_log_pathname()) and matches_regular_expression(current_log_pathname(), "([0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9])")) then ( v.date = $1; v.path = current_log_pathname(); ); if (matches_regular_expression(current_log_line(), '^([0-9][0-9]:[0-9][0-9]:[0-9][0-9])\\\\.[0-9][0-9]+ [0-9]* (.*)$')) then ( v.time = $1; v.message = $2; # if (matches_regular_expression(v.message, '^(ENQUEUER|QUEUE)-*[0-9]*\\\\(\\\\[([0-9]*)\\\\]\\\\) from <([^>]*)>, ([0-9]*) bytes ')) then ( # v.key = $2; # set_collected_field(v.key, 'operation', $1); # set_collected_field(v.key, 'source_address', $3); # set_collected_field(v.key, 'bytes_sent', $4); # ); if (matches_regular_expression(v.message, '^(QUEUE)\\\\(\\\\[([0-9]*)\\\\]\\\\) (.*)$')) then ( v.key = $2; # set_collected_field(v.key, 'operation', $1); v.remainder = $3; # When the queue is deleted, clear all fields because ids are recycled. #00:00:00.149 2 QUEUE([5344949]) deleted if (matches_regular_expression(v.remainder, '^deleted')) then ( set_collected_field(v.key, 'date_time', "clear"); # won't be accepted because date_time is corrupted rekey_collected_entry(v.key, "clear"); ); #00:00:03.827 2 QUEUE([5344950]) from , 1353 bytes (<1160539085.6569@sand.com>) else if (matches_regular_expression(v.remainder, '^from <([^>]*)>, ([0-9]*) bytes ')) then ( # set_collected_field(v.key, 'source_address', $1); set_collected_field(v.key, 'sender', $1); set_collected_field(v.key, 'bytes_sent', $2); ); else if (matches_regular_expression(v.remainder, '^header: (.*)$')) then ( v.header = $1; if (matches_regular_expression(v.header, '^From: (.*)$')) then ( v.sender = $1; if (matches_regular_expression(v.sender, '<([^>]*)>')) then ( v.sender = $1; #set_collected_field(v.key, 'sender', $1); ); set_collected_field(v.key, 'sender', v.sender); ); else if (matches_regular_expression(v.header, '^To: (.*)$')) then ( set_collected_field(v.key, 'recipient', $1); ); else if (matches_regular_expression(v.header, '^Subject: (.*)$')) then ( set_collected_field(v.key, 'subject', $1); ); ); ); # if QUEUE # 2009-09-10 - KBB - This duplicated another section and prevented an accept from happening. # # Handle DEQUEUER lines # else if (matches_regular_expression(v.message, '^(DEQUEUER) \\\\[([0-9]+)\\\\] (.*)$')) then ( # v.key = $2; # set_collected_field(v.key, 'operation', $1); # v.remainder = $3; ## if (matches_regular_expression(v.remainder, '^LOCAL\\\\((.*)\\\\) delivered$')) then ( ## set_collected_field(v.key, 'local_account', $1); ## ); ## else if (matches_regular_expression(v.remainder, '^SMTP\\\\(.*\\\\)(.*) relayed$')) then ( ## set_collected_field(v.key, 'local_account', $1); ## ); # ); # if DEQUEUER # 2009-09-07 - KBB - In most logs, these are followed by DEQUEUER/delivered lines. # The only exception I saw was a log with no relays in it. The local_account field # will be used to make sure this event isn't counted again at DEQUEUER/delivered. # Accept on ACCOUNT delivery lines else if (matches_regular_expression(v.message, '^ACCOUNT\\\\(([^)]+)\\\\) \\\\[([0-9]*)\\\\] delivered')) then ( v.account = $1; v.key = $2; set_collected_field(v.key, 'time', v.time); set_collected_field(v.key, 'date', v.date); set_collected_field(v.key, 'messages_sent', 1); set_collected_field(v.key, 'recipient', $1); set_collected_field(v.key, 'local_account', $1); # this is to check to see if this event has happened # set_collected_field(v.key, 'operation', 'ACCOUNT'); set_collected_field(v.key, 'action', 'delivered LOCAL'); set_collected_field(v.key, 'queue_id', v.key); accept_collected_entry(v.key, true); ); # 2009-09-10 - KBB - The backslashes at the end of two of the lines are probably # an artifact of the log being saved/sent improperly, but supporting anyway. #00:00:09.763 2 DEQUEUER [2885109] SYSTEM(here.there.everywhere.com)sarah.crewe@everywhere.com delivered: delivered via Automatic Rules #00:00:04.172 2 DEQUEUER [5344950] SYSTEM(tomcat) delivered: delivered via Automatic Rules #00:08:36.29 2 DEQUEUER [3817404] LOCAL(nilda.nongaala) delivered\ #00:40:33.72 2 DEQUEUER [3817488] SMTP(*)alert@notification.everywhere.com relayed\ #00:00:04.092 2 DEQUEUER [2885105] SMTP(here.there.everywhere.com)lfigleaf@elsewhere.everywhere.com relayed: relayed via here.there.everywhere.com #01:13:38.432 2 DEQUEUER [1171741] LIST(#ufficio.postale)group relayed: distributed to the group else if (matches_regular_expression(v.message, '^(D?E?QUEUER?) +\\\\[([0-9]+)\\\\] (.*)$')) then ( v.key = $2; # set_collected_field(v.key, 'operation', $1); v.remainder = $3; # Accept on delivered/relayed lines if (matches_regular_expression(v.remainder, '([A-Z]+)(\\\\([^ ]+) (delivered|relayed)')) then ( v.recipient = $2; v.action = $3; set_collected_field(v.key, 'action', $3 . " " . $1); if (matches_regular_expression(v.recipient, '\\\\(([^)]+)\\\\)(.*)?')) then ( if ($2 ne '' and $2 ne 'group') then ( v.recipient = $2; ); else ( v.recipient = $1; ); ); if ((v.action eq 'relayed') or ((v.action eq 'delivered') and (get_collected_field(v.key, 'local_account') ne v.recipient))) then ( set_collected_field(v.key, 'time', v.time); set_collected_field(v.key, 'date', v.date); set_collected_field(v.key, 'recipient', v.recipient); set_collected_field(v.key, 'messages_sent', 1); set_collected_field(v.key, 'queue_id', v.key); accept_collected_entry(v.key, true); ); set_collected_field(v.key, 'local_account', ''); # clear for next check ); ); # Handle POP lines else if (matches_regular_expression(v.message, '^(POP)-([0-9]*)(.*)$')) then ( v.key = $2; v.remainder = $3; set_collected_field(v.key, 'protocol', $1); # Handle connect lines if (matches_regular_expression(v.remainder, '^[^ ]* \\'(.*)\\' connected from \\\\[([^:]*):')) then ( set_collected_field(v.key, 'pop_account', $1); set_collected_field(v.key, 'source_host', $2); set_collected_field(v.key, 'pop_logins', 1); ); # Handle disconnected lines else if (matches_regular_expression(v.remainder, '^[^ ]* \\'(.*)\\' disconnected \\\\(\\\\[([^:]*):')) then ( set_collected_field(v.key, 'pop_account', $1); set_collected_field(v.key, 'source_host', $2); set_collected_field(v.key, 'time', v.time); set_collected_field(v.key, 'date', v.date); set_collected_field(v.key, 'queue_id', v.key); accept_collected_entry(v.key, false); ); # Handle message retrievals else if (matches_regular_expression(v.remainder, '^\\\\(\\\\[[^]]*\\\\]\\\\) [0-9]+ \\\\{[0-9]+\\\\} retrieved, ([0-9]*) bytes')) then ( set_collected_field(v.key, 'pop_messages_retrieved', get_collected_field(v.key, 'pop_messages_retrieved') + 1); set_collected_field(v.key, 'pop_bytes_retrieved', get_collected_field(v.key, 'pop_bytes_retrieved') + $1); ); # Handle message deletions else if (matches_regular_expression(v.remainder, '^\\\\(\\\\[[^]]*\\\\]\\\\) deleting ([0-9]+) message')) then ( set_collected_field(v.key, 'pop_messages_deleted', get_collected_field(v.key, 'pop_messages_deleted') + $1); ); # Handle other end lines # if (contains(v.remainder, 'proxy connection closed') or # contains(v.remainder, 'polling account') or # contains(v.remainder, 'retrieved')) then # accept_collected_entry(v.key, false); ); # POP # Handle IMAP lines else if (matches_regular_expression(v.message, '^(IMAP)-([0-9]*)(.*)$')) then ( v.key = $2; v.remainder = $3; set_collected_field(v.key, 'protocol', $1); # Handle connect lines if (matches_regular_expression(v.remainder, '^[^ ]* \\'(.*)\\' connected from \\\\[([^:]*):')) then ( set_collected_field(v.key, 'imap_account', $1); set_collected_field(v.key, 'source_host', $2); set_collected_field(v.key, 'imap_logins', 1); ); # Handle disconnected lines else if (matches_regular_expression(v.remainder, '^[^ ]* \\'(.*)\\' disconnected \\\\(\\\\[([^:]*):')) then ( set_collected_field(v.key, 'imap_account', $1); set_collected_field(v.key, 'source_host', $2); set_collected_field(v.key, 'time', v.time); set_collected_field(v.key, 'date', v.date); set_collected_field(v.key, 'queue_id', v.key); accept_collected_entry(v.key, false); ); ); # IMAP # Handle RPOP else if (matches_regular_expression(v.message, '^(RPOP)-([0-9]*)\\\\((.*)\\\\) polling account (.*) for ')) then ( v.key = $2; set_collected_field(v.key, 'protocol', $1); set_collected_field(v.key, 'source_host', $3); set_collected_field(v.key, 'pop_account', $4); ); # RPOP # If SMTPI #00:00:12.714 4 SMTPI-013598(here.there.com) cmd: RCPT To: else if (matches_regular_expression(v.message, '^SMTPI-([0-9]*)\\\\(([^)]*)\\\\) (.*)$')) then ( v.key = $1; set_collected_field(v.key, 'source_host', $2); v.remainder = $3; # Handle MAIL FROM if (matches_regular_expression(v.remainder, '^cmd: [Mm][Aa][Ii][Ll] [Ff][Rr][Oo][Mm]:(.*)$')) then ( v.sender = $1; if (matches_regular_expression(v.sender, '^(.*) [Ss][Ii][Zz][Ee]=([0-9]+)$')) then ( v.sender = $1; set_collected_field(v.key, 'bytes_sent', $2); ); if (matches_regular_expression(v.sender, '<([^>]*)>')) then set_collected_field(v.key, 'sender', $1); else set_collected_field(v.key, 'sender', v.sender); ); # MAIL FROM # Handle RCPT TO else if (matches_regular_expression(v.remainder, '^cmd: [Rr][Cc][Pp][Tt] [Tt][Oo]:(.*)$')) then ( v.recipient = $1; if (matches_regular_expression(v.recipient, '<([^>]*)>')) then set_collected_field(v.key, 'recipient', $1); else set_collected_field(v.key, 'recipient', v.recipient); ); # RCPT TO # Handle rekey #00:00:03.827 2 SMTPI-013596(coolcarrot.com) [5344950] received, 1353 bytes else if (matches_regular_expression(v.remainder, '^\\\\[([^]]*)\\\\] received[^,]*, ([0-9]*) bytes')) then ( set_collected_field(v.key, 'bytes_received', $1); rekey_collected_entry(v.key, $1); ); # ); # if SMTPI #00:00:14.487 4 PIPE processing carrot.sub # Handle first line of PIPE; remember previous key else if (matches_regular_expression(v.message, '^PIPE processing ([0-9]+)')) then ( v.previous_pipe_key = $1; ); # Handle second line of PIPE; rekey #00:00:14.487 2 PIPE [4397108] received in {Submitted\carrot.sub}, 1121 bytes else if (matches_regular_expression(v.message, '^PIPE \\\\[([0-9]+)\\\\] received, ([0-9]+) bytes')) then ( rekey_collected_entry(v.previous_pipe_key, $1); ); ); # if matches headers ` # Database fields database.fields = { date_time = "" day_of_week = "" hour_of_day = "" source_host = "" queue_id = "" protocol = "" # operation = "" action = "" sender = "" recipient = "" subject = "" pop_account = "" imap_account = "" } # database.fields # Log Filters log.filters = { # This filter sets size = msgsize if size is zero 1 = { label = "1" comment = "" value = "if (protocol eq 'POP') then '' else protocol = 'SMTP';" } # 1 } # log.filters database.numerical_fields = { messages_sent = { default = true entries_field = true } # message_sent bytes_sent = { type = "int" integer_bits = 64 display_format_type = "bandwidth" } # bytes_sent pop_logins = "" pop_messages_retrieved = "" pop_messages_deleted = "" pop_bytes_retrieved = { type = "int" integer_bits = 64 display_format_type = "bandwidth" } imap_logins = "" } # database.numerical_fields create_profile_wizard_options = { date_time_tracking = true # This shows which numerical fields are related to which non-numerical fields. database_field_associations = { pop_account = { pop_logins = true pop_messages_retrieved = true pop_messages_deleted = true pop_bytes_retrieved = true } imap_account = { imap_logins = true } # source_address = { # messages_sent = true # bytes_sent = true # } # local_account = { # messages_received = true # bytes_received = true # } # operation = { # messages_sent = true # bytes_sent = true # } action = { messages_sent = true bytes_sent = true } protocol = { messages_sent = true bytes_sent = true } sender = { messages_sent = true bytes_sent = true } recipient = { messages_sent = true bytes_sent = true } subject = { messages_sent = true bytes_sent = true } } # database_field_associations # How the reports should be grouped in the report menu report_groups = { date_time_group = "" } # report_groups } # create_profile_wizard_options } # communigate_pro