# Copyright (c) 2010 Flowerfire, Inc.  All Rights Reserved.
microsoft_exchange_2007_csv = {

  plugin_version = "1.1.3"

  info.1.manufacturer = "Microsoft"
  info.1.device = "Exchange Server 2007/2010"
  info.1.version.1 = "2007"
  info.1.version.2 = "2010"

  # 2007-08-16 - KBB - 1.0 - Initial creation. Note that this is a w3c-like format, but it is
  # comma separated, so it does not use w3c processing.
  # 2008-07-04 - GMF - 1.0.1 - Added tracking of recipient_count as a numerical field.
  # 2011-01-03 - GMF - 1.1 - Added support for Exchange Server 2010 message tracking logs
  # 2011-04-27 - GMF - 1.1.1 - Added support for IPv6 in Message Logs; added support for quoted
  # source_context with commas in it.
  # 2011-06-23 - 1.1.2 - MSG - Edited info lines.
  # 2011-08-19 - 1.1.3 - KBB - Corrected problems with info lines.

  # The name of the log format
  log.format.format_label = "Microsoft Exchange Server 2007/2010 Log Format (comma separated)"
  log.miscellaneous.log_data_type = "mail_server"
  log.miscellaneous.log_format_type = "mail_server"

  # Don't treat as w3c
  log.format.ignore_format_lines = true

  #Log-type: Agent Log
  #Fields: Timestamp,SessionId,LocalEndpoint,RemoteEndpoint,EnteredOrgFromIP,MessageId,P1FromAddress,P2FromAddresses,Recipient,NumRecipients,Agent,Event,Action,SmtpResponse,Reason,ReasonData
  #Log-type: Message Tracking Log
  #Fields: date-time,client-ip,client-hostname,server-ip,server-hostname,source-context,connector-id,source,event-id,internal-message-id,message-id,recipient-address,recipient-status,total-bytes,recipient-count,related-recipient-address,reference,message-subject,sender-address,return-path,message-info
  #Log-type: SMTP Receive Protocol Log
  #Fields: date-time,connector-id,session-id,sequence-number,local-endpoint,remote-endpoint,event,data,context
  #Log-type: SMTP Send Protocol Log
  #Fields: date-time,connector-id,session-id,sequence-number,local-endpoint,remote-endpoint,event,data,context

  # The log is in this format if any of the first ten lines match this regular expression
  log.format.autodetect_regular_expression = '^#Fields: (Timestamp,SessionId,LocalEndpoint,RemoteEndpoint,EnteredOrgFromIP,MessageId,P1FromAddress,P2FromAddresses|date-time,client-ip,client-hostname,server-ip,server-hostname,source-context,connector-id,source,event-id,internal-message-id,message-id,recipient-address|date-time,connector-id,session-id,sequence-number,local-endpoint,remote-endpoint,event,data,context)'

  # All log field parsing will be done using the parsing filters
  log.format.parse_only_with_filters = "true"

  # The format of dates and times in this log
  log.format.date_format = "auto"
  log.format.time_format = "auto"

  # Log fields
  log.fields = {

    date = ""
    time = ""
    action = ""
    agent = ""
    client_hostname = ""
    client_ip.type = "host"
    connector_id = ""
    context = ""
    data = ""
    entered_org_from_ip = ""
    event = ""
    event_id = ""
    internal_message_id = ""
    local_endpoint = ""
    message_id = ""
    message_info = ""
    message_subject = ""
    num_recipients = ""
    p1_from_address = ""
    p2_from_addresses = ""
    reason = ""
    reason_data = ""
    recipient = ""
    recipient_address = ""
    recipient_count = ""
    recipient_status = ""
    reference = ""
    related_recipient_address = ""
    remote_endpoint = ""
    return_path = ""
    sender_address = ""
    sequence_number = ""
    server_hostname = ""
    server_ip = ""
    session_id = ""
    smtp_response = ""
    source = ""
    source_context = ""
    total_bytes.type = "size"

    # Additional fields of Exchange Server 2010
    directionality = ""
    tenant_id = ""
    original_client_ip = ""
    original_server_ip = ""
    custom_data = ""

  } # log.fields


  # Log Parsing Filters
  log.parsing_filters.parse = `

v.line = current_log_line();

# Get date and time
if (matches_regular_expression(v.line, '^([0-9]{4}-[0-9]{2}-[0-9]{2})T([0-9]{2}:[0-9]{2}:[0-9]{2})\\\\.[^,]+,(.*)$')) then (
  set_collected_field('', 'date', $1);
  set_collected_field('', 'time', $2);
  v.line = $3;
);

# Handle Agent Log
#Fields: Timestamp,SessionId,LocalEndpoint,RemoteEndpoint,EnteredOrgFromIP,MessageId,P1FromAddress,P2FromAddresses,Recipient,NumRecipients,Agent,Event,Action,SmtpResponse,Reason,ReasonData
#2007-06-07T00:00:20.570Z,08C976940F1334BE,199.199.199.197:25,199.199.199.198:2217,199.199.199.199,,someone@here.co.uk,,someone@there.com,1,Recipient Filter Agent,OnRcptCommand,RejectCommand,550 5.1.1 User unknown,RecipientDoesNotExist,
if (matches_regular_expression(v.line, '^([0-9A-Z]+),([0-9.]+:[0-9]+),([0-9.]+:[0-9]+),([0-9.]+),([^,]*),([^,]*),([^,]*),("[^"]+"|[^,]*),("[^"]+"|[^,]*),("[^"]+"|[^,]*),([^,]*),("[^"]+"|[^,]*),("[^"]+"|[^,]*),("[^"]+"|[^,]*),("[^"]+"|[^,]*)')) then (

  set_collected_field('', 'session_id', $1);
  set_collected_field('', 'local_endpoint', $2);
  set_collected_field('', 'remote_endpoint', $3);
  set_collected_field('', 'entered_org_from_ip', $4);
  set_collected_field('', 'message_id', $5);
  set_collected_field('', 'p1_from_address', $6);
  set_collected_field('', 'p2_from_addresses', $7);
  set_collected_field('', 'recipient', $8);
  set_collected_field('', 'num_recipients', $9);
  set_collected_field('', 'agent', $10);
  set_collected_field('', 'event', $11);
  set_collected_field('', 'action', $12);
  set_collected_field('', 'smtp_response', $13);
  set_collected_field('', 'reason', $14);
  set_collected_field('', 'reason_data', $15);

  accept_collected_entry('', false);

); # Agent Log
  
# Handle Message Tracking Log
#Fields: date-time,client-ip,client-hostname,server-ip,server-hostname,source-context,connector-id,source,event-id,internal-message-id,message-id,recipient-address,recipient-status,total-bytes,recipient-count,related-recipient-address,reference,message-subject,sender-address,return-path,message-info
#2007-05-25T12:30:10.430Z,199.199.199.198,exchange2007.intra.theca.com.br,199.199.199.199,ntsthecam,,,STOREDRIVER,RECEIVE,2,<75DDC78CC68049479F5A90E92C16C9B205F3611F@exchange2007.here.there.com>,5900.someone@there.com;5900.who@there.com,,34167,2,,,NOTA DE CORRETAGEM,lanalang@there.com,clarkkent@there.com,03I:
#
# or with IPv6
# 2010-07-07T00:00:48.024Z,fe80::e123:dcc9:64b1:8f92%11,W0J23,,W0J96,"MDB:eb8766ce-5be6-4b0d-b79f-becfcec15261, Mailbox:e240f8fe-8b6f-4ecd-a82b-65278910f54c, Event:446465, MessageClass:IPM.Note, CreationTime:2010-07-07T00:00:47.041Z, ClientType:OWA",,STOREDRIVER,SUBMIT,,<140FDDC3521B734B851CAC962719E90901B08F06BC@W0J23.tcpd.gov.tw>,,,,,,,,LLEQENdtw@tcpd.gov.tw,,
#
# This doesn't work because of a Salang bug. The fix will be in 7.2.11.
# The problem is that multiple sets of quotes can be in the same field, without comma between them.
# 2011-04-27 - GMF - In the IPv6 example above, we can't use [^,]+ for source_context, because there's a double-quoted field with commas in it.  Changed it to ("[^"]*"|[^,]*)
else if (matches_regular_expression(v.line, '^([0-9.]*|[0-9a-f:%]*),([^,]*),([0-9.]*),([^,]*),("[^"]*"|[^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),(".*"|[^,]*),([0-9]*),([0-9]*),([^,]*),([^,]*),(".*"|[^,]*),([^,]*),([^,]*),([^,]*)$')) then (

  set_collected_field('', 'client_ip', $1);
  set_collected_field('', 'client_hostname', $2);
  set_collected_field('', 'server_ip', $3);
  set_collected_field('', 'server_hostname', $4);
  set_collected_field('', 'source_context', $5);
  set_collected_field('', 'connector_id', $6);
  set_collected_field('', 'source', $7);
  set_collected_field('', 'event_id', $8);
  set_collected_field('', 'internal_message_id', $9);
  set_collected_field('', 'message_id', $10);
  set_collected_field('', 'recipient_address', $11);
  set_collected_field('', 'recipient_status', $12);
  set_collected_field('', 'total_bytes', $13);
  set_collected_field('', 'recipient_count', $14);
  set_collected_field('', 'related_recipient_address', $15);
  set_collected_field('', 'reference', $16);
  set_collected_field('', 'message_subject', $17);
  set_collected_field('', 'sender_address', $18);
  set_collected_field('', 'return_path', $19);
  set_collected_field('', 'message_info', $20);

  accept_collected_entry('', false);

); # Message Tracking Log 

# Handle Message Tracking Log from Exchange 2010
#Fields: date-time,client-ip,client-hostname,server-ip,server-hostname,source-context,connector-id,source,event-id,internal-message-id,message-id,recipient-address,recipient-status,total-bytes,recipient-count,related-recipient-address,reference,message-subject,sender-address,return-path,message-info,directionality,tenant-id,original-client-ip,original-server-ip,custom-data
else if (matches_regular_expression(v.line, '^([0-9.]*),([^,]*),([0-9.]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),(".*"|[^,]*),([0-9]*),([0-9]*),([^,]*),([^,]*),(".*"|[^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*)$')) then (

  set_collected_field('', 'client_ip', $1);
  set_collected_field('', 'client_hostname', $2);
  set_collected_field('', 'server_ip', $3);
  set_collected_field('', 'server_hostname', $4);
  set_collected_field('', 'source_context', $5);
  set_collected_field('', 'connector_id', $6);
  set_collected_field('', 'source', $7);
  set_collected_field('', 'event_id', $8);
  set_collected_field('', 'internal_message_id', $9);
  set_collected_field('', 'message_id', $10);
  set_collected_field('', 'recipient_address', $11);
  set_collected_field('', 'recipient_status', $12);
  set_collected_field('', 'total_bytes', $13);
  set_collected_field('', 'recipient_count', $14);
  set_collected_field('', 'related_recipient_address', $15);
  set_collected_field('', 'reference', $16);
  set_collected_field('', 'message_subject', $17);
  set_collected_field('', 'sender_address', $18);
  set_collected_field('', 'return_path', $19);
  set_collected_field('', 'message_info', $20);
  set_collected_field('', 'directionality', $21);
  set_collected_field('', 'tenant_id', $22);
  set_collected_field('', 'original_client_ip', $23);
  set_collected_field('', 'original_server_ip', $24);
  set_collected_field('', 'custom_data', $25);

  accept_collected_entry('', false);

); # Message Tracking Log (Exchange Server 2010)

# Handle SMTP Receive/Send Protocol logs
#Fields: date-time,connector-id,session-id,sequence-number,local-endpoint,remote-endpoint,event,data,context
#2007-06-16T21:32:22.870Z,NTSTHECAM\Default NTSTHECAM,08C97DA67B889902,3,199.199.199.198:25,199.199.199.199:18042,<,EHLO merakmailserver.hello.com.br,
#2007-06-16T21:32:25.714Z,NTSTHECAM\Default NTSTHECAM,08C97DA67B889903,2,199.199.199.199:25,199.199.199.198:1182,>,"220 mail.hello.com.br Microsoft ESMTP MAIL Service ready at Sat, 16 Jun 2007 18:32:25 -0300",
#2007-05-26T00:05:50.034Z,Daisy Exchange2007,08C96D40C04B5E9D,0,,199.199.199.199:25,*,,attempting to connect
#2007-05-26T00:05:50.096Z,Daisy Exchange2007,08C96D40C04B5E9D,2,199.199.199.198:1461,199.199.199.199:25,<,554 xyz00smtp.goodbye.cn,
#2007-05-26T00:06:11.693Z,Daisy Exchange2007,08C96D40C04B5E9E,1,,199.199.199.199:25,*,,"Failed to connect. Error Code: 10060, Error Message: A connection attempt failed because the connected party did not properly respond after a period of time, or established connection failed because connected host has failed to respond"

else if (matches_regular_expression(v.line, '^([^,]*),([^,]+),([0-9]*),([0-9.:]*),([0-9.:]*),([^,]*),("[^"]*"|[^,]*),("[^"]*"|[^,]*)$')) then (

  set_collected_field('', 'connector_id', $1);
  set_collected_field('', 'session_id', $2);
  set_collected_field('', 'sequence_number', $3);
  set_collected_field('', 'local_endpoint', $4);
  set_collected_field('', 'remote_endpoint', $5);
  set_collected_field('', 'event', $6);
  set_collected_field('', 'data', $7);
  set_collected_field('', 'context', $8);

  accept_collected_entry('', false);

); # SMTP Receive/Send Protocol

`

  # Database fields
  database.fields = {

    date_time = ""
    hour_of_day = ""
    day_of_week = ""
    action = ""
    agent = ""
    client_hostname = ""
    client_ip = ""
    connector_id = ""
    context = ""
    data = ""
    entered_org_from_ip = ""
    event = ""
    event_id = ""
    internal_message_id = ""
    local_endpoint = ""
    message_id = ""
    message_info = ""
    message_subject = ""
    num_recipients = ""
    p1_from_address = ""
    p2_from_addresses = ""
    reason = ""
    reason_data = ""
    recipient = ""
    recipient_address = ""
    recipient_status = ""
    reference = ""
    related_recipient_address = ""
    remote_endpoint = ""
    return_path = ""
    sender_address = ""
    sequence_number = ""
    server_hostname = ""
    server_ip = ""
    session_id = ""
    smtp_response = ""
    source = ""
    source_context = ""

    # Additional fields of Exchange Server 2010
    directionality = ""
    tenant_id = ""
    original_client_ip = ""
    original_server_ip = ""
    custom_data = ""

  } # database.fields


  log.filters = {

    mark_entry = {
      label = '$lang_admin.log_filters.mark_entry_label'
      comment = '$lang_admin.log_filters.mark_entry_comment'
      value = 'events = 1;'
    } # mark_entry

  } # log.filters


  database.numerical_fields = {

    events = {
      default = true
      requires_log_field = false
      entries_field = true
    } # events

    total_bytes = {
      default = true
      type = "int"
      integer_bits = 64
      display_format_type = "bandwidth"
    } # total_bytes

    recipient_count = ""

  } # database.numerical_fields


  create_profile_wizard_options = {

    # How the reports should be grouped in the report menu
    report_groups = {
      date_time_group = ""
      agent_group = {
        session_id = true
        local_endpoint = true
        remote_endpoint = true
        entered_org_from_ip = true
        message_id = true
        p1_from_address = true
        p2_from_addresses = true
        recipient = true
        num_recipients = true
        agent = true
        event = true
        action = true
        smtp_response = true
        reason = true
        reason_data = true
      }
      message_tracking_group = {
        client_ip = true
        client_hostname = true
        server_ip = true
        server_hostname = true
        source_context = true
        connector_id = true
        source = true
        event_id = true
        internal_message_id = true
        message_id = true
        recipient_address = true
        recipient_status = true
        recipient_count = true
        related_recipient_address = true
        reference = true
        message_subject = true
        sender_address = true
        return_path = true
        message_info = true
        directionality = true
        tenant_id = true
        original_client_ip = true
        original_server_ip = true
        custom_data = true
      }
      send_receive_group = {
        connector_id = true
        session_id = true
        sequence_number = true
        local_endpoint = true
        remote_endpoint = true
        event = true
        data = true
        context = true
      }
    } # report_groups

  } # create_profile_wizard_options

} # microsoft_exchange_2007_csv