# Copyright (c) 2010 Flowerfire, Inc.  All Rights Reserved.
array_spx_squid = {

  plugin_version = "2.6.3"

  info.1.manufacturer = "Array Networks"
  info.1.device = "SPX 3000 WELF/Squid combined"
  info.1.version.1 = "Rel.SP.8.4.4.2"
  info.1.version.2 = "Rel.SP.8.4.4.1 Build 4"

  # 2008-06-04 - 1.0 - GMF - Initial implementation.  This implements support for only the urlaccesslog (there are at least 4 other formats which are not yet supported)
  # 2010-08-12 - 2.0 - Benson - Fixed for full WELF log format support.
  # 2010-08-13 - 2.5 - Benson - Adds mixed Squid proxy log format support.
  # 2010-08-20 - 2.5.1 - Benson - Update for older firmware SP.8.4.4.1 Build 4.
  # 2014-05-12 - 2.5.2 - GMF - Added support for variant formats.
  # 2014-05-12 - 2.5.3 - GMF - Added support for duration field
  # 2014-05-12 - 2.5.4 - GMF - Added extraction of session_duration from the msg field
  # 2014-05-12 - 2.5.5 - GMF - Added User Agent analysis
  # 2014-06-19 - 2.6 - GMF - Added unnormalized msg field; added successful/failed logins; added unique users
  # 2014-06-20 - 2.6.1 - GMF - Extended successful/failed logins to handle "authentication failed" and "authentication succeeded"
  # 2014-06-20 - 2.6.2 - GMF - Fixed parsing of successful/failed logins to handle double quotes and trailing @
  # 2014-07-10 - 2.6.3 - ILD - Changed type of msg field to normalized string so it will appear in Log Detail (note: performance will suffer for large datasets)

  # The name of the log format
  log.format.format_label = "Array Networks SPX WELF & Squid mixed Log Format"
  log.miscellaneous.log_data_type = "syslog_required"  
  log.miscellaneous.log_format_type = "firewall"

  # The log is in this format if any of the first ten lines match this regular expression
  # 2007-05-11	16:58:47	Local0.Info	12.34.56.78	May 11 17:03:41 AN id=ArraySP time="2007-5-11 17:03:41" fw=AN pri=6 user=someone type=mgmt msg="CLI cmd "sh run" success code 0"
  # 2010-08-10	14:24:44	Local0.Info	10.1.1.12	Aug 10 15:03:01 AN id=ArrayOS time="2010-8-10 15:03:01" fw=AN pri=6 vpn=intranet proto=http src=10.1.0.3 sport=47562 dstname=localhost arg=/ op=GET result=302 type=vpn msg="Request on port 80 redirected to port 443"
  # 2010-08-10	14:24:19	Local0.Info	10.1.1.12	Aug 10 15:02:36 AN AN_SQUID_LOG 1281452556.369 729 10.1.0.3 TCP_MISS/200 1075 GET /exchweb/img/newitem.gif - DIRECT/10.1.1.10 -
  # 2010-08-20	15:09:12	Local7.Info	192.168.0.11	Aug 20 15:10:54 GMT(+0000) SPX3000 id=ArrayOS time="2010-8-20 15:10:54" fw=SPX3000 pri=6 user=array src=192.168.5.28 sport=1059 type=mgmt msg="CLI cmd "show log config" success code 0"

  log.format.autodetect_regular_expression = "(id=ArraySP|id=ArrayOS|AN_SQUID_LOG) "

  # All log field parsing will be done using the parsing filters
  log.format.parse_only_with_filters = "true"

  # Log fields
  log.fields = {

    id = ""
    log_type = ""
    fwtime = ""
    fw = ""
    pri = ""
    vpn = ""
    user = ""
    proto = ""
    src = ""
    action = ""
    sport = ""
    dst = ""
    dport = ""
    dstname = ""
    arg = ""
    op = ""
    result = ""
    rcvd = ""
    sent = ""
    type = ""
    agent = ""
   
    msg = ""

    duration = ""
    session_duration = ""
    events = ""

    successful_logins = ""
    failed_logins = ""

  } # log.fields


  # Log Parsing Filters
  log.parsing_filters.parse = `

  # Remove timestamp from start:
  #May 08 14:24:03 P AN_WELF_LOG:id=ArrayOS time="2014-5-8 14:24:03" timezone=PDT(-0700) fw=P pri=6 vpn=site user=user1 proto=http src=10.1.231.34 dstname=localhost:9090 arg=/query/clientres?_uname=user1&_sessid=site+0aa1dda0_aec6fa01e3f6e6da633ef7696a8f2144 op=GET agent="Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.131 Safari/537.36" result=200 sent=658 duration=0.000 msg="cache:TCP_MISS  peer:DIRECT/127.0.0.1"@
  if (matches_regular_expression(v.syslog_message, '^[A-Z][a-z][a-z] [0-9 ][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9] [^ ]+ (.*)$')) then
    v.syslog_message = $1;

  # 2010-08-10	14:24:19	Local0.Info	10.1.1.12	Aug 10 15:02:36 AN AN_SQUID_LOG 1281452556.369 729 10.1.0.3 TCP_MISS/200 1075 GET /exchweb/img/newitem.gif - DIRECT/10.1.1.10 -
  # 2010-08-12	14:00:22	Local0.Info	10.1.1.12	Aug 12 14:39:24 AN AN_SQUID_LOG 1281623964.048 1 10.1.0.3 TCP_MISS/304 341 GET /bugzilla/skins/contrib/Mine/index.css  - DIRECT/10.3.0.70 -
# SQUID log type
#if (matches_regular_expression(v.syslog_message, 'AN_SQUID_LOG [0-9.]+ [0-9]+ ([^ ]*) ([A-Z_ ]*)/([0-9]*) ([0-9]*) ([A-Z]*) ([^ ]*)[ |  ]([^ ]*) ([A-Z_]*)/([^ ]*) (.*)')) then (
if (matches_regular_expression(v.syslog_message, 'AN_SQUID_LOG [0-9.]+ [0-9]* ([^ ]*) ([A-Z_ ]*)/([0-9]*) ([0-9]*) ([^ ]*) ([^ ]*)[ |  ](.*)')) then (
  set_collected_field('', 'src', $1);
  set_collected_field('', 'action', $2);
  set_collected_field('', 'result', $3);
  set_collected_field('', 'rcvd', $4);
  set_collected_field('', 'op', $5);
  set_collected_field('', 'arg', $6);
  v.message = $7;
  if (matches_regular_expression(v.message, '([^ ]+) ([A-Z_]*)/([^ ]*) (.*)')) then (
    set_collected_field('', 'user', $1);
    set_collected_field('', 'msg', $2);
    set_collected_field('', 'dst', $3);
    set_collected_field('', 'type', $4);
  );
  set_collected_field('', 'log_type', 'SQUID');
  set_collected_field('', 'events', 1);
  accept_collected_entry('', false);
);
# WELF log type
#May 08 14:24:03 P AN_WELF_LOG:id=ArrayOS time="2014-5-8 14:24:03" timezone=PDT(-0700) fw=P pri=6 vpn=site user=user1 proto=http src=10.1.231.34 dstname=localhost:9090 arg=/query/clientres?_uname=user1&_sessid=site+0aa1dda0_aec6fa01e3f6e6da633ef7696a8f2144 op=GET agent="Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.131 Safari/537.36" result=200 sent=658 duration=0.000 msg="cache:TCP_MISS  peer:DIRECT/127.0.0.1"@
else if (matches_regular_expression(v.syslog_message, '[^ ]+ (id=.*) msg=(.*)$')) then (
  collect_listed_fields('', $1, ' ', '=', 'time=fwtime');

  # Convert duration to milliseconds
  set_collected_field('', 'duration', get_collected_field('', 'duration') * 1000.0);

  set_collected_field('', 'msg', $2);

  # Define log type
  if (matches_regular_expression($2, 'Authorized access')) then (set_collected_field('', 'log_type', 'HTTP'););
  else if (matches_regular_expression($2, 'CLI cmd')) then (set_collected_field('', 'log_type', 'CLI'););
  else if (matches_regular_expression($2, 'Request on port 80 redirected to port 443')) then (set_collected_field('', 'log_type', 'HTTP'););
  else if (matches_regular_expression($2, 'URL')) then (set_collected_field('', 'log_type', 'HTTP'););
  else if (matches_regular_expression($2, 'Authentication successful')) then (set_collected_field('', 'log_type', 'HTTP'););
  else if (matches_regular_expression($2, 'valid session')) then (set_collected_field('', 'log_type', 'HTTP'););
  else if (matches_regular_expression($2, 'Session has expired')) then (set_collected_field('', 'log_type', 'HTTP'););
  else if (matches_regular_expression($2, 'idletime')) then (set_collected_field('', 'log_type', 'HTTP'););
  else if (matches_regular_expression($2, 'logged')) then (set_collected_field('', 'log_type', 'HTTP'););
  else if (matches_regular_expression($2, 'Backend connection')) then (set_collected_field('', 'log_type', 'HTTP'););
  else if (matches_regular_expression($2, 'clientapp:')) then (set_collected_field('', 'log_type', 'ClientApp'););
  else if (matches_regular_expression($2, 'Killed an expired session')) then (set_collected_field('', 'log_type', 'MGMT'););
  else if (matches_regular_expression($2, 'vpn:')) then (set_collected_field('', 'log_type', 'VPN'););
  else if (matches_regular_expression($2, 'AAA authentication')) then (set_collected_field('', 'log_type', 'Mail Proxy'););
  else if (matches_regular_expression($2, 'SSL [client|driver|handshake|backend|tunnel]')) then (set_collected_field('', 'log_type', 'SSL'););
  else if (matches_regular_expression($2, 'Invalid request')) then (set_collected_field('', 'log_type', 'HTTP'););
  else if (matches_regular_expression($2, 'Authentication failed (.*)')) then (
     while (
       matches_regular_expression($1, 'host check') or
       matches_regular_expression($1, 'credentials rejected') or
       matches_regular_expression($1, 'internal') or
       matches_regular_expression($1, 'sessions') or
       matches_regular_expression($1, 'username') or
       matches_regular_expression($1, 'SSL') 
     ) (set_collected_field('', 'log_type', 'HTTP');)
  );
  else (set_collected_field('', 'log_type', 'Misc'));

  # convert date and time
  if (matches_regular_expression(get_collected_field('', 'fwtime'), '^([0-9-]+) ([0-9:]+)$')) then (
    set_collected_field('', 'date', $1);
    set_collected_field('', 'time', $2);
  );
  set_collected_field('', 'events', 1);
  accept_collected_entry('', false);
);

# WELF log type
#May 08 14:24:03 P AN_WELF_LOG:id=ArrayOS time="2014-5-8 14:24:03" timezone=PDT(-0700) fw=P pri=6 vpn=site user=user1 proto=http src=10.1.231.34 dstname=localhost:9090 arg=/query/clientres?_uname=user1&_sessid=site+0aa1dda0_aec6fa01e3f6e6da633ef7696a8f2144 op=GET agent="Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.131 Safari/537.36" result=200 sent=658 duration=0.000 msg="cache:TCP_MISS  peer:DIRECT/127.0.0.1"@
# Other WELF style lines
#2014-05-08 14:23:04       Local0.Info     10.10.151.210   May 08 14:20:39 P id=ArrayOS time="2014-5-8 14:20:39" timezone=PDT(-0700) fw=P pri=6 user=array type=mgmt msg="CLI cmd "log host 10.1.2.116 514 udp 0" success code 0"@
else if (matches_regular_expression(v.syslog_message, '^[^ ]+:(id=.*) msg=(.*)$') or
         matches_regular_expression(v.syslog_message, '^(id=.*) msg=(.*)$')) then (
  collect_listed_fields('', $1, ' ', '=', 'time=fwtime');
  v.msg = $2;

  # Remove quotes, and trailing @
  if (matches_regular_expression(v.msg, '^"(.*)"@$')) then
    v.msg = $1;
  else if (matches_regular_expression(v.msg, '^"(.*)"$')) then
    v.msg = $1;

  set_collected_field('', 'msg', v.msg);

  # Convert duration to milliseconds
  set_collected_field('', 'duration', get_collected_field('', 'duration') * 1000.0);

  # convert date and time
  if (matches_regular_expression(get_collected_field('', 'fwtime'), '^([0-9-]+) ([0-9:]+)$')) then (
    set_collected_field('', 'date', $1);
    set_collected_field('', 'time', $2);
  );

  # Extract session duration from the message
#2014-05-08 14:33:27	Local0.Info	10.10.151.210	May 08 14:31:01 P id=ArrayOS time="2014-5-8 14:31:01" timezone=PDT(-0700) fw=P pri=6 vpn=site user=user1 src=10.1.231.34 sport=6966 dport=80 dstname=localhost arg=/prx/000/http/localhost/logout type=vpn msg="User user1 logged out successfully, session id AA1DDA0, duration 418 seconds."
  if (matches_regular_expression(v.msg, '^"User ([^ ]+) logged out successfully, session id ([^,]+), duration ([0-9]+)')) then 
    set_collected_field('', 'session_duration', $3);

  set_collected_field('', 'events', 1);
  accept_collected_entry('', false);
);

# 2010-08-11	14:02:41	Local0.Notice	10.1.1.12	Aug 11 14:41:20 AN DNS: All nameservers - down.
# Other log types
else if (matches_regular_expression(v.syslog_message, '[^ ] DNS: (.*)$')) then (
  set_collected_field('', 'log_type', 'DNS');
  set_collected_field('', 'msg', $1);
  set_collected_field('', 'events', 1);
  accept_collected_entry('', false);
);
`

  # Database fields
  database.fields = {

    # 2014-06-19 - GMF - Removed by request of Array Networks (ThreadID:1308487); useless
#    id = ""
#    log_type = ""

    #fwtime = ""
    fw = ""
    pri = ""
    vpn = ""
    user = ""
    proto = ""
    src = ""
    sport = ""
    dst = ""
    dport = ""
    dstname = ""
    action = ""
    arg = ""
    op = ""
    result = ""
    type = ""

    msg = {
      # 2014-07-10 - ILD - Changed type to normalized string so it will appear in Log Detail (note: performance will suffer for large datasets)
#      type = "unnormalized_string"
      type = "string"
      index = false
      add_default_xref_table = false
    } # msg

  } # database.fields

  # Log Filters
  log.filters = {

#INFO    Apr 18 14:14:10 P id=ArrayOS time="2014-4-18 14:14:10" timezone=PDT(-0700) fw=P pri=6 user=superuser src=10.1.231.20 sport=33548 dst=10.10.151.210 dport=8888 type=mgmt msg="Log in by WebUI."
#INFO    Apr 18 14:16:57 P id=ArrayOS time="2014-4-18 14:16:57" timezone=PDT(-0700) fw=P pri=6 user=test src=10.1.231.20 sport=47628 dst=10.10.151.210 dport=8888 type=mgmt msg="Log in failed by WebUI."

    categorize_login = {
      label = "Categorize Logins"
      comment = "This determines whether this event is a successful login, or an unsuccessful login (or neither)"
      value = `
if (matches_regular_expression(msg, '^(Log in by |authentication succeeded)')) then
  successful_logins = 1;
else if (matches_regular_expression(msg, '^(Log in failed by |authentication failed)')) then
  failed_logins = 1;`
      requires_fields = {  
        successful_logins = true
        failed_logins = true
      }
    } # categorize_login

  } # log.filters

  database.numerical_fields = {

    events = {
      default = true
      requires_log_field = false
      entries_field = true
    } # events
    rcvd = {
      default = true
      type = "int"
      integer_bits = 64
      display_format_type = "bandwidth"
    }
    sent = {
      default = true
      type = "int"
      integer_bits = 64
      display_format_type = "bandwidth"
    }
    duration = {
      integer_bits = 64
      display_format_type = "duration_milliseconds"
    } # duration
    session_duration = {
      integer_bits = 64
      display_format_type = "duration_compact"
    } # session_duration

    unique_users = {
      log_field = "user"
      type = "unique"
    } # unique_users

    successful_logins = ""
    failed_logins = ""

  } # database.numerical_fields

  create_profile_wizard_options = {

    # How the reports should be grouped in the report menu
    report_groups = {
      date_time_group = ""

      admin_messages = {
        label = "Admin Messages"
        filter = "type = 'mgmt'"
        columns = {
          0.field_name = "msg"
          1.field_name = "events"
        }
        sort_by = events
      } # admin_messages

    } # report_groups

    snapons = {
      
      # Add the standard reports
      add_standard_reports = {
        name = "add_standard_reports"
	label = "add_standard_reports"
	snapon = "add_standard_reports"
      } # add_standard_reports

      user_agent_analysis = {
        snapon = "user_agent_analysis"
        name = "user_agent_analysis"
        label = "$lang_admin.snapons.user_agent_analysis.label"
        parameters = {
          user_agent_field.parameter_value = "agent"
          page_views_field.parameter_value = "accesses"
        } # parameters
      } # user_agent_analysis

      geo_location = {
	snapon = "geo_location"
	name = "geo_location"
	  label = "$lang_admin.snapons.geo_location.label"
          parameters = {
           ip_address_field.parameter_value = "src"
          } # parameters

      } # geo_location

    } # snapons

  } # create_profile_wizard_options

} # array_spx_squid