# Copyright (c) 2010 Flowerfire, Inc.  All Rights Reserved.
postfix = {

  plugin_version = "1.13"

  # 2006-08-09 - GMF - 1.1beta - Added tracking of SpamAssassin information, when present.
  # This was created to attempt to fix a number of problems with the old format,
  # without changing the existing format, as that would take too long and may not succeed.
  # A 'ground up' approach has been undertaken. This has now been updated to v7 Salang parse
  # filter and renamed to the current label as the first choice for this log (the old postfix has
  # been removed).
  # 2007-06-12 - gas - 1.2beta - added support for a variant (Postfix: 2.4.0)
  # 2007-09-13 - 1.2 - KBB - renumbered per new beta policy and changed name from beta_postfix.cfg
  # 2007-10-25 - 1.3 - KBB - fixed bug where from and to not being captured on blocked messages
  # 2007-10-26 - 1.4 - gas - added support for relay-ip containing port info
  # (relay=mail.domain.com[123.21.123.21]:25)
  # 2008-01-28 - 1.5 - MSG for GMF - In 'Handle to= lines', moved 'v.message = $2;' above second regexp
  # match line. Added support for letters other than A-F in message ID.
  # 2008-02-07 - 1.6 - KBB - Renamed message_id field to queue_id and added code to put the value
  # of message-id (message-id=<748735E9.0203024@nowhere.com>) field message_id. Some code clean-up.
  # Set v.status at top to fix bug. Use set_collected for message-id to mid mapping to save memory.
  # 2008-04-10 - 1.7 - GMF - Added support for Amavis lines.
  # 2008-04-10 - 1.8 - GMF - Added support for MailScanner lines.
  # 2009-04-07 - 1.9 - GMF - Added filters to suppress message_id and queue_id, and switched to rand_hash
  # 2010-09-09 - 1.10 - GMF - Added detection of Brightmail variant
  # 2010-09-09 - 1.11 - GMF - Added support for random info at the end of the Client host section above in reject lines.
  # 2010-09-17 - 1.12 - GMF - Changed the label to specify "Brightmail Gateway" instead of just "Brightmail"
  # 2011-01-25 - 1.13 - GAS - Added support for sasl_method and sasl_user

  # log file format info, latest changes
  info.1.manfacturer = "Postfix"
  info.1.device = "mail server"
  info.1.version.1 = "2.4.0"

  info.2.manfacturer = "Symantec"
  info.2.device = "Brightmail Gateway"
  info.2.version.1 = ""


  # The name of the log format
  log.format.format_label = "Postfix or Brightmail Gateway Log Format"
  log.miscellaneous.log_data_type = "syslog_required"
  log.miscellaneous.log_format_type = "mail_server"

  # The log is in this format if there is a match this regular expression
  log.format.autodetect_regular_expression = "postfix|outbound-mta/[a-z]+[[][0-9]+[]]"

  # Search this many lines for the regex for auto-detect (this is set so high because of some variants
  # that produce many lines at the top of the file that do not contain our test "postfix",
  # for performance, this could be reduced without much danger).
  log.format.autodetect_lines = "100"

  # All log field parsing will be done using the parsing filters
  log.format.parse_only_with_filters = "true"

  # This discards uncollected entries after 10000 lines
  log.format.collected_entry_lifespan = "10000"
  log.format.discard_expired_entries = true

  # Log fields
  log.fields = {

    from = {
      type = "hierarchical"
      hierarchy_dividers = "@"
      left_to_right = false
      leading_divider = false
    }

    to = {
      type = "hierarchical"
      hierarchy_dividers = "@"
      left_to_right = false
      leading_divider = false
    }

    size = ""
    user = ""
    rbl_list = ""
    amavis_result = ""
    mailscanner_result = ""
    client_hostname = ""
    client_ip.type = "host"
    relay_hostname = ""
    relay_ip = ""
    direction = ""
    status = ""
    sc_status = ""
#    nrcpt = ""
#    counter = ""
    queue_id = ""

    message_id = {
        itemnums_hash_function = "rand_sum"
    }

    messages_delivered = ""
    messages_processed = ""
    messages_scanned = ""
    messages_blocked = ""
    messages_expired = ""
    messages_delivered = ""
    messages_bounced = ""
    bytes_delivered = ""
    bytes_processed = ""
    bytes_blocked = ""
    bytes_expired = ""
    bytes_delivered = ""
    bytes_bounced = ""

    spam_detected = ""
    sasl_method = ""
    sasl_username = ""
    

  } # log.fields

#  log.filter_initialization = `
#node message_id_info;
#v.message_id_infos = '';
#node message_id_infos = 'v.message_id_infos';
#`

  # Log Parsing Filters
  log.parsing_filters.parse = `
v.status = "";

if (matches_regular_expression(v.syslog_message, '^[^ ]+\\[*[^]]*\\]*:? ([0-9A-Z]+|NOQUEUE|spamd): (.*)$')) then (

  v.key = $1;
  v.message = $2;

  set_collected_field(v.key, 'date', get_collected_field('', 'date'));
  set_collected_field(v.key, 'time', get_collected_field('', 'time'));
  set_collected_field(v.key, 'logging_device', get_collected_field('', 'logging_device'));
  set_collected_field(v.key, 'syslog_priority', get_collected_field('', 'syslog_priority'));

  if (v.key ne 'NOQUEUE') then
    set_collected_field(v.key, 'queue_id', v.key);

  if (matches_regular_expression(v.message, '^client=([^[]*)\\[([^]]*)\\]$')) then (
    if ($1 ne 'localhost') then (
      set_collected_field(v.key, 'client_hostname', $1);
      set_collected_field(v.key, 'client_ip', $2);
    );
  );

  if (matches_regular_expression(v.message, '^client=([^[]*)\\[([^]]*)\\], sasl_method=([^,]+), sasl_username=([^$]+)$')) then (
    if ($1 ne 'localhost') then (
      set_collected_field(v.key, 'client_hostname', $1);
      set_collected_field(v.key, 'client_ip', $2);
      set_collected_field(v.key, 'sasl_method', $3);
      set_collected_field(v.key, 'sasl_username', $4);
    );
  );

  # Handle blocked messages
  else if (matches_regular_expression(v.message, '^(reject): RCPT from ([^[]*)\\[([^]]*)\\]: ([0-9][0-9][0-9]) [^;]+;(.*)$')) then (
    set_collected_field(v.key, 'status', $1);
    v.client_hostname = $3;
    set_collected_field(v.key, 'client_hostname', v.client_hostname);
    set_collected_field(v.key, 'client_ip', $3);
    set_collected_field(v.key, 'sc_status', $4);
    v.message = $5;
    if (matches_regular_expression(v.message, '^ *Client host \\[[0-9.]+\\] blocked using ([^;]+);[^;]*;* (.*)$')) then (
      set_collected_field(v.key, 'rbl_list', $1);
      v.message = $2;
    );
    # 2010-09-09 - 1.11 - GMF - Added support for random info at the end of the Client host section above in reject lines.
    if (matches_regular_expression(v.message, ' *from=<([^>]*)> to=<([^>]*)> ')) then (
      set_collected_field(v.key, 'from', $1);
      set_collected_field(v.key, 'to', $2);
      set_collected_field(v.key, 'messages_blocked', 1);
      set_collected_field(v.key, 'bytes_blocked', get_collected_field(v.key, 'size'));
      accept_collected_entry(v.key, true);
    );
  );

  # Handle from= lines
  else if (matches_regular_expression(v.message, '^from=([^,]*),(.*)$')) then (

    v.from = $1;
    v.message = $2;

    if (matches_regular_expression(v.from, '^<([^>]*)>$')) then
      v.from = $1;

    set_collected_field(v.key, 'from', v.from);
    if (matches_regular_expression(v.message, '^ size=([0-9]*)(.*)$')) then (
      set_collected_field(v.key, 'size', $1);
    )

    else if (matches_regular_expression(v.message, '^ status=([^,]*),')) then (
      v.status = $1;
      set_collected_field(v.key, 'status', $1);
    );
    if (v.status eq 'expired') then (
      set_collected_field(v.key, 'messages_expired', 1);
      set_collected_field(v.key, 'bytes_expired', get_collected_field(v.key, 'size'));
    );
    set_collected_field(v.key, 'messages_processed', 1);
    set_collected_field(v.key, 'bytes_processed', get_collected_field(v.key, 'size'));
    accept_collected_entry(v.key, true);

  ); # from=

  # Handle to= lines
  else if (matches_regular_expression(v.message, '^to=([^,]*),(.*)$')) then (

    v.to = $1;
    v.message = $2;

    set_collected_field(v.key, 'date', get_collected_field('', 'date'));
    set_collected_field(v.key, 'time', get_collected_field('', 'time'));
    set_collected_field(v.key, 'logging_device', get_collected_field('', 'logging_device'));
    set_collected_field(v.key, 'syslog_priority', get_collected_field('', 'syslog_priority'));

    if (matches_regular_expression(v.to, '^<([^>]*)>$')) then
      v.to = $1;

    set_collected_field(v.key, 'to', v.to);
    if (matches_regular_expression(v.message, '^ orig_to=([^,]*),(.*)$')) then (
      v.message = $2;
    );
    if (matches_regular_expression(v.message, '^ relay=([^,]*),(.*)$')) then (
      v.relay_ip = $1;
      v.message = $2;
      # relay_ip contains both hostname and IP, strip that out if no port
      if (matches_regular_expression(v.relay_ip, '^([^[]*)\\[([0-9.]*)\\]$')) then (
        set_collected_field(v.key, 'relay_hostname', $1);
        v.relay_ip = $2;
      );
      # relay_ip contains both hostname and IP (and sometimes port), strip that out if there is a port
      else if (matches_regular_expression(v.relay_ip, '^([^[]*)\\[([0-9.]*)\\]:[0-9]+$')) then (
        set_collected_field(v.key, 'relay_hostname', $1);
        v.relay_ip = $2;
      );
      set_collected_field(v.key, 'relay_ip', v.relay_ip);
    );
    if (matches_regular_expression(v.message, '^ delay=([^,]*),(.*)$')) then (
      set_collected_field(v.key, 'delay', $1);
      v.message = $2;
    );
    
    # if status=sent
    if (matches_regular_expression(v.message, ' status=([^, ]*) (.*)$')) then (

      v.status = $1;
      v.message = $2;

      set_collected_field(v.key, 'status', v.status);

      if (matches_regular_expression(v.message, '^\\\\(host [^ ]+ said: (.*)\\\\)$')) then
        v.message = $1;

      if (matches_regular_expression(v.message, '^\\\\(([0-9]+) ([^)]*)\\\\)')) then (
        v.message = $2;
        set_collected_field(v.key, 'response_code', $2);
        set_collected_field(v.key, 'response_message', $3);
      );

    ); # if status=
    
# 1.6 - KBB - removed ^ above for same result - why process these formats separately? -
# whatever is between status= and the beginning is thrown away here anyway
#    # 1.2beta - duplicated the if status=sent section above and changed the regex so that ' status=' does not have to start a line - gas
#    else if (matches_regular_expression(v.message, ' status=([^, ]*) (.*)$')) then (
#      v.status = $1;
#      set_collected_field(v.key, 'status', v.status);
#      v.message = $2;
#      if (matches_regular_expression(v.message, '^\\(host [^ ]+ said: (.*)$')) then
#        v.message = $1;
#      if (matches_regular_expression(v.message, '^\\(*([0-9]+) ([^)]*)\\)')) then (
#        set_collected_field(v.key, 'response_code', $2);
#        set_collected_field(v.key, 'response_message', $3);
#      );
#      v.message = $2;
#    ); # 1.2beta - duplicated the if status=sent section above and changed the regex so that ' status=' does not have to start a line - gas
    
    if (matches_regular_expression(v.message, 'queued as ([0-9A-Z]+)\\)')) then (
      set_collected_field(v.key, 'size', get_collected_field($1, 'size'));
      if (get_collected_field($1, 'client_hostname') eq '(empty)') then
        set_collected_field(v.key, 'client_hostname', get_collected_field($1, 'client_hostname'));
      if (get_collected_field($1, 'client_ip') eq '(empty)') then
        set_collected_field(v.key, 'client_ip', get_collected_field($1, 'client_ip'));
    );
    set_collected_field(v.key, 'messages_processed', 0);
    set_collected_field(v.key, 'bytes_processed', 0);
    if (v.status eq 'sent') then (
      set_collected_field(v.key, 'messages_delivered', 1);
      set_collected_field(v.key, 'bytes_delivered', get_collected_field(v.key, 'size'));
    );
    else if ((v.status eq 'bounce') or (v.status eq 'bounced')) then (
      set_collected_field(v.key, 'messages_bounced', 1);
      set_collected_field(v.key, 'bytes_bounced', get_collected_field(v.key, 'size'));
    );
    accept_collected_entry(v.key, true);

  ); # if to=

  # Handle message-id lines
  else if (matches_regular_expression(v.message, '^message-id=<([^>]+)>')) then (

    v.mid = $1;
    set_collected_field(v.key, 'message_id', v.mid);

    v.from = get_collected_field(v.key, 'from');
    v.to = get_collected_field(v.key, 'to');

    #message_id_info = subnode_by_name(message_id_infos, replace_all(replace_all($1, '$', '_'), '.', '_'));
    #set_subnode_value(message_id_info, "from", replace_all(v.from, '.', '_'));
    #set_subnode_value(message_id_info, "to", replace_all(v.from, '.', '_'));

    # If these are empty, they'll be empty anyway if they aren't saved here.
    if (v.from ne '(empty)') then (
      set_collected_field(v.mid, 'from', v.from);
#      echo("FROM: " . v.from);
    );
    if (v.to ne '(empty)') then (
      set_collected_field(v.mid, 'to', v.to);
#      echo("TO: " . v.to);
    );
  ); # if message-id

  # Handle "hold" lines
  else if (matches_regular_expression(v.message, '^hold: .*; from=<([^>]+)> to=<([^>]+)>')) then (
    set_collected_field(v.key, 'from', $1);
    set_collected_field(v.key, 'to', $2);
  ); # hold

  # Handle spamd lines
  else if (matches_regular_expression(v.message, '^result: Y (.*)$')) then (

    v.message = $1;

    if (matches_regular_expression(v.message, 'mid=<([^>]+)>')) then (

      v.mid = $1;

      set_collected_field(v.key, 'message_id', v.mid);

      #message_id_info = subnode_by_name(message_id_infos, replace_all($1, '.', '_'));
      #set_collected_field(v.key, 'from', node_value(subnode_by_name(message_id_info, "from")));
      #set_collected_field(v.key, 'to', node_value(subnode_by_name(message_id_info, "to")));

      set_collected_field(v.key, 'from', get_collected_field(v.mid, 'from'));
      set_collected_field(v.key, 'to', get_collected_field(v.mid, 'to'));
    );

    set_collected_field(v.key, 'spam_detected', 1);
    accept_collected_entry(v.key, true);
  );

);

# Handle amavis lines
# e.g. amavis[16921]: (16121-09) Passed CLEAN, [12.34.56.78] [98.76.54.32] <someone@somewhere.com> -> <someoneelse@elsewhere.com>, Message-ID: <001a01c8303c$abe0ebce0$52d5494b@Someone>, mail_id: 3xnJA6gLw2HS, Hits: 1.441, queued_as: CA123C1234B, 1251 ms
# e.g. Nov 26 09:26:44 mail amavis[20582]: (20582-06) Blocked SPAM, [24.222.0.30] [216.167.248.203] <alyssa_lewin@nhusd.k12.ca.us> -> <alichang@edina.k12.mn.us>, Message-ID: <012201c8303d$60feaaa0$cbf8a7d8@NAQSQ>, mail_id: Ra+--ULDuZbi, Hits: 6.586, 1179 ms

if (matches_regular_expression(v.syslog_message, '^amavis[[][0-9]+[]]: [(][0-9-]+[)] ([^ ]+) ([^,]+), [[]([0-9.]+)[]] [[]([0-9.]+)[]] <([^>]*)> -> <([^>]*)>, Message-ID: <([^>]*)>, mail_id: ([^ ]+), Hits: ([^,]+), (.*)')) then (
  v.remainder = $10;
  v.amavis_result = $1;
  v.amavis_reason = $2;
  v.source_ip = $3;
  v.from = $5;
  v.to = $6;
  v.message_id = $7;

  # If there is a queued_as line, we can use that as the key, and we don't need to accept yet.
  # If there is no queued_as line, we need to set and accept this as a single entry.
  if (matches_regular_expression(v.remainder, 'queued_as: ([^,]+),')) then (
    v.key = $1;
  );
  else (
    v.key = "";
    set_collected_field(v.key, 'source_ip', v.source_ip);
    set_collected_field(v.key, 'from', v.from);
    set_collected_field(v.key, 'to', v.to);
    set_collected_field(v.key, 'message_id', v.message_id);
  );

  set_collected_field(v.key, 'amavis_result', v.amavis_result . ' ' . v.amavis_reason);

  if ((v.amavis_result eq "Blocked") and (v.amavis_reason eq "SPAM")) then
    set_collected_field(v.key, 'spam_detected', 1);

  if (v.amavis_result eq "Blocked") then 
    set_collected_field(v.key, 'messages_blocked', 1);

  set_collected_field(v.key, 'messages_scanned', 1);

  # If key is "", that means there was no queue ID above, so we need to accept this as a separate entry.
  if (v.key eq "") then
    accept_collected_entry(v.key, false);

); # if amavis

# Handle MailScanner lines
# e.g. MailScanner[17221]: Content Checks: Detected and have disarmed web bug, phishing tags in HTML message in BAB62C78093.94DA3 from bounce-12345-987654@abc.def.com
if (matches_regular_expression(v.syslog_message, '^MailScanner[[][0-9]+[]]: (.*)')) then (
  v.mailscanner_message = $1;

  # Ignore Requeue lines
  if (matches_regular_expression(v.mailscanner_message, '^Requeue: ')) then (
  );

  # Ignore New Batch lines
  else if (matches_regular_expression(v.mailscanner_message, '^New Batch: ')) then (
  );

  # Ignore "Virus and Content Scanning: Starting" lines
  else if (matches_regular_expression(v.mailscanner_message, '^Virus and Content Scanning: Starting')) then (
  );

  else (
    set_collected_field('', 'mailscanner_result', v.mailscanner_message);
    set_collected_field('', 'messages_scanned', 1);
    accept_collected_entry('', true);
  );

); # if mailscanner


` # parse

  log.filters = {

    suppress_message_id = {
      label = "$lang_admin.log_filters.suppress_message_id_label"
      comment = "$lang_admin.log_filters.suppress_message_id_comment"
      value = `message_id = "[omitted]"`
    } # suppress_message_id

    suppress_queue_id = {
      label = "$lang_admin.log_filters.suppress_queue_id_label"
      comment = "$lang_admin.log_filters.suppress_queue_id_comment"
      value = `queue_id = "[omitted]"`
    } # suppress_queue_id

  } # log.filters
  
  # Database fields
  database.fields = {

    from = {
      itemnums_hash_function = "rand_sum"
    }
    to = {
      itemnums_hash_function = "rand_sum"
    }
    client_hostname = ""
    client_ip = ""
    relay_hostname = ""
    relay_ip = ""
    status = ""
    sc_status = ""
    rbl_list = ""
    amavis_result = ""
    mailscanner_result = ""
    queue_id = {
      itemnums_hash_function = "rand_sum"
    }

    message_id = {
      itemnums_hash_function = "rand_sum"
    }

    sasl_method = ""
    sasl_username = ""


  } # database.fields

  database.numerical_fields = {

    messages_delivered = {
      default = true
    }
    messages_processed = {
      default = true
    }
    messages_blocked = ""
    messages_scanned = ""
    messages_expired = ""
    messages_bounced = ""

    bytes_delivered = {
      type = "float"
      display_format_type = "bandwidth"
      default = true
    }

    bytes_processed = {
      type = "float"
      display_format_type = "bandwidth"
      default = true
    }

    bytes_blocked = {
      type = "float"
      display_format_type = "bandwidth"
    }

    bytes_expired = {
      type = "float"
      display_format_type = "bandwidth"
    }

    bytes_bounced = {
      type = "float"
      display_format_type = "bandwidth"
    }

    spam_detected = ""

#    nrcpt = {
##      label = "$lang_stats.field_labels.nrcpt"
#      default = false
#      requires_log_field = false
#      type = "int"
#      display_format_type = "integer"
#      entries_field = true
#    } # nrcpt
#
#    size = {
#      label = "$lang_stats.field_labels.size"
#      default = false
#      requires_log_field = true
#      log_field = "size"
#      type = "float"
#      display_format_type = "bandwidth"
#    } # size

  } # database.numerical_fields

  create_profile_wizard_options = {

    # How the reports should be grouped in the report menu
    report_groups = {
      from = true
      to = true
      rbl_list = true
      client_hostname = true
      client_ip = true
      relay_hostname = true
      relay_ip = true
      status = true
      sc_status = true
      sasl_method = true
      sasl_username = true

    } # report_groups

  } # create_profile_wizard_options

} # postfix