microsoft_isa_w3c = {

  plugin_version = "2.3.1"

  info.1.manfacturer = "Microsoft"
  info.1.device = "ISA Server"
  info.1.version.0 = "2000"
  info.1.version.1 = "2004"
  info.1.version.2 = "2006"

  # 2007-05-29 - 1.1 - KBB - Fixed bug in and simplified autodetection. Added new fields to groups.
  # 2008-04-18 - 1.1.1 - KBB - Put derived fields organization, domain and isp into
  # 2008-05-23 - 1.1.2 - GMF - Changed hash function to rand_sum for filterinfo visitor_demographics_group.
  # 2008-09-05 - 2.0 - GMF - Changed W3C parsing to use expression, to handle variants not supported by standard W3C parser.
  # 2008-12-19 - 2.1 - GMF - Fixed delimiter to tab; otherwise user-agent with spaces embedded, unquoted, is split into multiple fields.
  #                        - Added support for c-ip and cs-uri field names
  # 2009-01-15 - 2.2 - GMF - Added filterinfo log field--v8 profile creation fails without it.
  # 2010-06-14 - 2.3 - MSG - Added support for "ISA Sever 2006" in the #Software line.
  # 2010-10-28 - 2.3.1 - GMF - Tweaked autodetection to support cases where the "(R)" is missing in the Software line

  # The name of the log format
  log.format.format_label = "Microsoft ISA Server Log Format (W3C)"
  log.miscellaneous.log_data_type = "generic_w3c"  
  log.miscellaneous.log_format_type = "firewall"

  log.format.ignore_format_lines = true

  # 2008-12-19 - GMF - tabs between fields!  Otherwise user-agent, unquoted, gets split into a bunch of fields
  log.format.field_separator = "	"

  # The log is in this format if any of the first ten lines match this regular expression
  # 2010-10-28 - 2.3.1 - GMF - Tweaked autodetection to support cases where the "(R)" is missing in the Software line
  log.format.autodetect_regular_expression = '(#Software: Microsoft [(]?R?[)]? ?Internet Security and Acceleration Server |#Software: Microsoft [(]R[)] ISA Server )'

# Note: This did not work with (R). Parentheses not special in starts_with, so backslashes not needed.
#  log.format.autodetect_expression = `
#starts_with(volatile.log_data_line, '#Software: Microsoft') and
#(starts_with(volatile.log_data_line, '#Software: Microsoft\\(R\\) Internet Security and Acceleration Server ') or
# starts_with(volatile.log_data_line, '#Software: Microsoft Internet Security and Acceleration Server '))
#`

  # The format of dates and times in this log
  log.format.date_format = "yyyy-mm-dd"
  log.format.time_format = "hh:mm:ss"

  # Don't track these fields as discrete database fields
#  auto_setup.omit_database_fields = "sc_bytes,cs_bytes,time_taken"


  # This handles #Fields lines, and creates log and database fields from them
  # e.g. #Fields: computer	date	time	IP protocol	source	destination	original client IP	source network	destination network	action	status	rule	application protocol	bytes sent	bytes sent intermediate	bytes received	bytes received intermediate	connection time	connection time intermediate	username	agent	session ID	connection ID

  # 2008-12-29 - GMF - AND A DIFFERENT FORMAT:
  #Software: Microsoft Internet Security and Acceleration Server 2006
  #Version: 2.0
  #Date: 2008-11-17 00:00:05
  # e.g. #Fields: c-ip	cs-username	c-agent	date	time	s-computername	cs-referred	r-host	r-ip	r-port	time-taken	cs-bytes	sc-bytes	cs-protocol	s-operation	cs-uri	s-object-source	sc-status	rule	FilterInfo	cs-Network	sc-Network	error-info	action	AuthenticationServer

  log.filter_preprocessor = `
if (matches_regular_expression(current_log_line(), '^#Fields: (.*)$')) then (
  string fields = $1;
  string fieldname;
  v.logfieldindex = 1;
  string numerical_fields = "profiles." . internal.profile_name . ".database.numerical_fields";

  # This subroutine creates a database field
  subroutine(create_database_field(string fieldname), (
    debug_message("create_database_field(" . fieldname . ")\n");
    string databasefieldpath = "profiles." . internal.profile_name . ".database.fields." . fieldname;
    (databasefieldpath . "") = "";
    node databasefield = databasefieldpath;
#    set_subnode_value(databasefield, "label", fieldname);
    databasefield;
  ));

  subroutine(create_log_field(string fieldname, string type, bool withindex), (
    debug_message("create_log_field(" . fieldname . "; type=" . type . ")\n");
    string logfieldpath = "profiles." . internal.profile_name . ".log.fields." . fieldname;
    (logfieldpath . "") = "";
    node logfield = logfieldpath;
#    set_subnode_value(logfield, "label", fieldname);
    if (withindex) then (
      set_subnode_value(logfield, "index", v.logfieldindex);
debug_message("Set index of " . fieldname . " to " . v.logfieldindex . "\n");
      v.logfieldindex++;
    );
    set_subnode_value(logfield, "subindex", 0);
    if (type ne '') then
      set_subnode_value(logfield, "type", type);
    logfield;
  ));

  # Assume there isn't a localtime field until we see one.
  v.parse_localtime = false;

  # Extract the fields on at a time
  while (matches_regular_expression(fields, '^([^	]+)	(.*)$')) (

    string unconverted_fieldname = $1;
    fields = $2;

    # Clean up the field name
    fieldname = '';
    for (int i = 0; i < length(unconverted_fieldname); i++) (
      string c = lowercase(substr(unconverted_fieldname, i, 1));
      if (!matches_regular_expression(c, '^[a-z0-9]$')) then
        c = '_';
      fieldname .= c;
    );
    while (matches_regular_expression(fieldname, '^(.*)_$'))
      fieldname = $1;

debug_message("fieldname=" . fieldname . "\n");

    # Get the log field type
    string log_field_type = '';
    if ((fieldname eq 'original_client_ip') or (fieldname eq "c_ip")) then
      log_field_type = 'host';
    if (fieldname eq 'cs_uri') then
      log_field_type = 'page';

    # Create the log field
    create_log_field(fieldname, log_field_type, true);

    # If we're creating a profile, create the database fields too.
    if (node_exists("volatile.creating_profile")) then (

      # Handle localtime by creating date_time and derived database fields
#      if (fieldname eq "date") then (
##        create_log_field('date', '', false);
##        create_log_field('time', '', false);
#        create_database_field('date_time');
#        create_database_field('day_of_week');
#        create_database_field('hour_of_day');
##        ("profiles." . internal.profile_name . ".log.parsing_filters.parse_localtime.disabled") = false;
#      ); # if localtime

      # Handle date by creating date_time and derived database fields
      if (fieldname eq "date") then (
#        create_log_field('localtime', '', false); # placeholder - 7/Nov/2006 - KBB
        create_database_field('date_time');
        create_database_field('day_of_week');
        create_database_field('hour_of_day');
#        ("profiles." . internal.profile_name . ".log.parsing_filters.parse_localtime.disabled") = true;
      ); # if date

      else if (fieldname eq "time") then (
        create_database_field('date_time');
        create_database_field('day_of_week');
        create_database_field('hour_of_day');
#        ("profiles." . internal.profile_name . ".log.parsing_filters.parse_localtime.disabled") = true;
      ); # if time

      # Create derived IP fields
      else if ((fieldname eq "original_client_ip") or
               (fieldname eq "c_ip")) then (
        create_database_field(fieldname);
        create_database_field('location');
        create_database_field('organization');
        create_database_field('isp');
        create_database_field('domain');
      );

      # Create derived page fields
      else if (fieldname eq "cs_uri") then (
        create_database_field(fieldname);
        create_database_field('file_type');
      );
  
      # Don't add a database field for numerical fields
      else if (subnode_exists(numerical_fields, fieldname)) then (
        debug_message("Not adding numerical field: " . fieldname . "\n");
      );
  
      # Create a normal database field
      else (
        create_database_field(fieldname);
      );

    ); # if creating profile
 
  ); # while another field

  # Don't parse the #Fields line as a data line
  'reject';

); # if #Fields

# Don't parse any other # lines as data lines
else if (starts_with(current_log_line(), '#')) then (
  'reject';
);
`

  # Get search engine and search phrase information from the referrer field (before it gets simplified).
  log.parsing_filters.compute_se_sp = {
    value = `
if (get_search_engine_info(referrer)) then (
  search_engine = volatile.search_engine;
  search_phrase = volatile.search_phrase;
);
`
    requires_fields = {
      referrer = true
    }
  } # compute_se_sp


  # Log Filters
  log.filters = {

    simplify_referrer = {
      label = "$lang_admin.log_filters.simplify_referrer_label"
      comment = "$lang_admin.log_filters.simplify_referrer_comment"
      value = "if (referrer eq '-') then referrer = '(no referrer)' else if (matches_regular_expression(referrer, '^([^:]+://[^/]+/)')) then referrer = $1 . '(omitted)'"
      requires_fields = {
        referrer = true
      }
    } # simplify_referrer

    internal_referrer = {
      label = "$lang_admin.log_filters.internal_referrer_label"
      comment = "$lang_admin.log_filters.internal_referrer_comment"
      value = "if (contains(referrer, 'mydomain.com/')) then referrer = '(internal referrer)';"
      disabled = true
      requires_fields = {
        referrer = true
      }
    } # internal_referrer

    eliminated_request_id = {
      label = "Eliminated request ID"
      comment = "This discards the Req ID value to simplify the database"
      value = "if (matches_regular_expression(filterinfo, '^Req ID: (.*)$')) then filterinfo = 'Req ID [omitted]'"
      disabled = false
      requires_fields = {
        filterinfo = true
      }
    } # internal_referrer

#    set_page_for_worm = {
#      label = "$lang_admin.log_filters.set_page_for_worm_label"
#      comment = "$lang_admin.log_filters.set_page_for_worm_comment"
#      value = "if (starts_with(worm, '(')) then '' else page = '(worm)';"
#      requires_fields = {
#        page = true
#     }
#    } # set_page_for_worm


    # This filter tacks the URL parameters ("URL query") onto the end of the URL field
    empty_uri_query = {
      label = "$lang_admin.log_filters.empty_uri_query_label"
      comment = "$lang_admin.log_filters.empty_uri_query_comment"
      value = "if (cs_uri_query eq '-') then cs_uri_query = '(empty)';"
      disabled = true
      requires_fields = {
        cs_uri_query = true
      }
    } # empty_uri_query

    1 = {
      label = "1"
      comment = ""
      value = "if (cs_uri_query ne '(empty)') then url = '' . url . '?' . cs_uri_query;"
      requires_fields = {
        cs_uri_query = true
      }
    } # 1

    remove_query = {
      label = "$lang_admin.log_filters.remove_query_label"
      comment = "$lang_admin.log_filters.remove_query_comment"
      value = "if (contains(cs_uri, '?')) then cs_uri = substr(cs_uri, 0, index(cs_uri, '?') + 1) . '(parameters)';"
      requires_fields = {
        cs_uri = true
      }
    } # remove_query


    detect_page_views = {
      label = '$lang_admin.log_filters.detect_page_views_label'
      comment = '$lang_admin.log_filters.detect_page_views_comment'
      value = "if ((file_type eq 'JPEG') or (file_type eq 'JPG') or (file_type eq 'GIF') or (file_type eq 'ICO') or (file_type eq 'PNG') or (file_type eq 'CSS') or (file_type eq 'SWF') or (file_type eq 'JS')) then page_views = 0; else page_views = 1;"
      requires_fields = {
        file_type = true
      }
    } # detect_page_views

    simplify_url = {
      label = "$lang_admin.log_filters.simplify_url_label"
      comment = "$lang_admin.log_filters.simplify_url_comment"
      value = "if (matches_regular_expression(cs_uri, '^([^:]+://[^/]+/)')) then cs_uri = $1 . '(omitted)'"
      requires_fields = {
        cs_uri = true
      }
    } # simplify_url

    strip_non_page_views = {
      label = '$lang_admin.log_filters.strip_non_page_views_label'
      comment = '$lang_admin.log_filters.strip_non_page_views_comment'
      value = "if (page_views == 0) then cs_uri = substr(cs_uri, 0, last_index(cs_uri, '/') + 1) . '(nonpage)';"
      requires_fields = {
        cs_uri = true
      }
    } # strip_non_page_views

    mark_entry = {
      label = '$lang_admin.log_filters.mark_entry_label'
      comment = '$lang_admin.log_filters.mark_entry_comment'
      value = 'hits = 1;'
    } # mark_entry

  } # log.filters

  log.field_options = {

    sessions_page_field = "cs_uri"
    sessions_visitor_id_field = "c_ip"
    sessions_event_field = "page_views"

  } # log.field_options


  # 2008-05-23 - GMF - use rand_sum for filterinfo, which is huge and ugly
  database.fields.filterinfo.itemnums_hash_function = "rand_sum"

  # 2009-01-15 - GMF - If we have a filterinfo database field, we need a filterinfo log field.
  log.fields.filterinfo = ""

  database.numerical_fields = {

    hits = {
      label = "$lang_stats.field_labels.hits"
      default = false
      requires_log_field = false
      type = "int"
      display_format_type = "integer"
      entries_field = true
    } # hits

    page_views = {
      label = "$lang_stats.field_labels.page_views"
      default = true
      requires_log_field = false
      type = "int"
      display_format_type = "integer"
    } # page_views

    visitors = {
      label = "$lang_stats.field_labels.visitors"
      default = false
      requires_log_field = true
      log_field = "c_ip"
      type = "unique"
      display_format_type = "integer"
    } # visitors

    sc_bytes = {
      label = "$lang_stats.field_labels.sc_bytes"
      default = false
      requires_log_field = false
      type = "float"
      display_format_type = "bandwidth"
    }

    cs_bytes = {
      label = "$lang_stats.field_labels.cs_bytes"
      default = false
      requires_log_field = false
      type = "float"
      display_format_type = "bandwidth"
    }

    bytes_sent = {
      type = "float"
      display_format_type = "bandwidth"
    }

    bytes_received = {
      type = "float"
      display_format_type = "bandwidth"
    }

    bytes_sent_intermediate = {
      type = "float"
      display_format_type = "bandwidth"
    }

    bytes_received_intermediate = {
      type = "float"
      display_format_type = "bandwidth"
    }

    time_taken = {
      label = $lang_stats.field_labels.time_taken
      default = false
      requires_log_field = true
      type = int
      display_format_type = duration_milliseconds
    } # time_taken

    connection_time = {
      requires_log_field = true
      display_format_type = duration_milliseconds
    } # connection_time

    connection_time_intermediate = {
      requires_log_field = true
      display_format_type = duration_milliseconds
    }

  } # database.numerical_fields


  create_profile_wizard_options = {

    # How the reports should be grouped in the report menu
    report_groups = {
      date_time_group = ""
      content_group = {
        file_type = true
        cs_uri = true
        cs_uri_stem = true
        cs_uri_query = true
        cs_mime_type = true
      }
      visitor_demographics_group = {
        original_client_ip = true
        c_ip = true
        domain_description = true
        location = true
        cs_username = true
        organization = true
        domain = true
        isp = true
      }
      visitor_systems_group = {
        screen_dimensions = true
        screen_depth = true
        web_browser = true
        operating_system = true
      }
      referrer_group = {
        referrer = true
        search_engine = true
        search_phrase = true
        referrer_description = true
        search_phrase_by_search_engine = true
      }
      server_group = {
        s_sitename = true
        s_computername = true
        s_ip = true
        s_port = true
        r_dns = true
        r_ip = true
        r_host = true
        r_port = true
        cs_host = true
        s_object_source = true
        s_svcname = true
      }
      other_group = {
        cs_version = true
        worm = true
        spider = true
        cs_method = true
        sc_status = true
        sc_substatus = true
        sc_win32_status = true
        cs_cookie = true
        cs_protocol = true
        cs_transport = true
        s_operation = true
        sc_authenticated = true
        sessionid = true
        connectionid = true
        rule_1 = true
        rule_2 = true
        s_cache_info = true
        rule = true
        filterinfo = true
        cs_network = true
        sc_network = true
        error_info = true
        action = true
      }
    } # report_groups

  } # create_profile_wizard_options

} # microsoft_isa_w3c