# Copyright (c) 2013 Flowerfire, Inc. All Rights Reserved. iis_advanced_logging_module = { plugin_version = "1.3" info.1.manufacturer = "Microsoft" info.1.device = "IIS Advanced Logging Module" info.1.version = "1.0" # 2013-01-16 - 1.0 - GMF - Initial implementation--based on iisweb # 2013-02-24 - 1.1 - GMF - Switched to log.filter_preprocessor, to allow support for BeginRequest-UTC and EndRequest-UTC fields, which do not follow standard W3C: they appear as single fields in the header, but as two fields in the data. # 2013-02-28 - 1.2 - GMF - Turned off derived fields in log.filter_preprocessor, since snapons do all the derivations for us now. # 2013-03-08 - 1.3 - GMF/MSG - Edited filter preprocessor so as not to track query or cookie. # The name of the log format log.format.format_label = "IIS Advanced Logging Module Log Format" log.miscellaneous.log_data_type = "generic_w3c" log.miscellaneous.log_format_type = "web_server" # The log is in this format if any of the first ten lines match this regular expression log.format.autodetect_regular_expression = "^#Software: IIS Advanced Logging Module" # Don't track these fields as discrete database fields. Don't add sc_bytes as a database field here, because it will be added by the web_server_package snapon. auto_setup.omit_database_fields = "cs_uri_query,cs_cookie,sc_bytes,cs_bytes" log.format.ignore_format_lines = "true" # 2013-02-23 - GMF - Transplanted this from iisweb # This handles #Fields lines, and creates log and database fields from them log.filter_preprocessor = ` if (matches_regular_expression(current_log_line(), '^#Fields: +([^ ].*)$')) then ( string fields = $1; string fieldname; v.logfieldindex = 1; string numerical_fields = "profiles." . internal.profile_name . ".database.numerical_fields"; # This subroutine creates a database field subroutine(create_database_field(string fieldname, int top, int bottom), ( #echo("create_database_field: " . fieldname); # debug debug_message("create_database_field(" . fieldname . ")\n"); string databasefieldpath = "profiles." . internal.profile_name . ".database.fields." . fieldname; (databasefieldpath . "") = ""; node databasefield = databasefieldpath; if (top ne 0) then set_subnode_value(databasefield, "suppress_top", top); if (bottom ne 0) then set_subnode_value(databasefield, "suppress_bottom", bottom); databasefield; )); subroutine(create_log_field(string fieldname, string type, bool withindex), ( debug_message("create_log_field(" . fieldname . "; type=" . type . ")\n"); string logfieldpath = "profiles." . internal.profile_name . ".log.fields." . fieldname; (logfieldpath . "") = ""; node logfield = logfieldpath; if (withindex) then ( set_subnode_value(logfield, "index", v.logfieldindex); v.logfieldindex++; ); set_subnode_value(logfield, "subindex", 0); if (type ne '') then set_subnode_value(logfield, "type", type); logfield; )); # Assume there isn't a localtime field until we see one. v.parse_localtime = false; # Get the log format separator. Assume at least two fields. # 2010-02-01 - GMF - Changed the default from " " to "" (any whitespace). With " " as the default, data which # has a spaces in the header, but tabs in the data, doesn't parse, because this assumes space # separators. Truly generic W3C header parsing appears to be impossible, which is why we can do # per-plugin parsing now; but that's just moved the problem here, where we'll still have to # fight it forever. Hopefully, "" gives better coverage to the universe of W3C variants than " ". # log.format.field_separator = " "; log.format.field_separator = ""; if (matches_regular_expression(fields, '^[^ ]+ [^ ]+')) then ( log.format.field_separator = " "; ); # Extract the fields on at a time while (matches_regular_expression(fields, '^([^ ]+)[ ](.*)$') or matches_regular_expression(fields, '^([^ ]+)$')) ( string unconverted_fieldname = $1; fields = $2; # Clean up the field name fieldname = ''; for (int i = 0; i < length(unconverted_fieldname); i++) ( string c = lowercase(substr(unconverted_fieldname, i, 1)); if (!matches_regular_expression(c, '^[a-z0-9]$')) then c = '_'; fieldname .= c; ); while (matches_regular_expression(fieldname, '^(.*)_$')) fieldname = $1; # Get the log field type string log_field_type = ''; if (fieldname eq 'cs_referrer') then ( fieldname = 'cs_referer'; ); if (fieldname eq 'cs_uri') then ( fieldname = 'cs_uri_stem'; ); # 2013-02-28 - GMF - Don't do any of this special "type" stuff anymore; it's all handled by snapons # if (fieldname eq 'cs_referer') then ( # log_field_type = 'url'; # ); # else if (fieldname eq 'cs_uri_stem') then ( # log_field_type = 'page'; # ); # else if (fieldname eq 'cs_user_agent') then ( # log_field_type = 'agent'; # ); # else if (fieldname eq 'c_ip') then ( # log_field_type = 'host'; # ); # 2013-02-24 - GMF - The beginrequest_utc and endrequest_utc fields appear as single fields in the header, but are actually two separate fields ("YYYY-MM-DD HH:MM:SS" without quotes) in the log data. Create two fields when we see them. else if (fieldname eq "beginrequest_utc") then ( create_log_field('beginrequest_utc_date', 'flat', true); fieldname = "beginrequest_utc_time"; log_field_type = "flat"; ); else if (fieldname eq "endrequest_utc") then ( create_log_field('endrequest_utc_date', 'flat', true); fieldname = "endrequest_utc_time"; log_field_type = "flat"; ); # Create the log field create_log_field(fieldname, log_field_type, true); if (fieldname eq "localtime") then v.parse_localtime = true; # If we're creating a profile, create the database fields too. if (node_exists("volatile.creating_profile")) then ( # Handle localtime by creating date_time and derived database fields if (fieldname eq "localtime") then ( create_log_field('date', '', false); create_log_field('time', '', false); create_database_field('date_time', 0, 0); create_database_field('day_of_week', 0, 0); create_database_field('hour_of_day', 0, 0); ); # if localtime # 2013-03-08 - GMF - Don't track query or cookie either if ((fieldname eq "sc_bytes") or (fieldname eq "cs_bytes") or (fieldname eq "cs_uri_query") or (fieldname eq "cs_cookie")) then ( ); # Handle date by creating date_time and derived database fields else if (fieldname eq "date") then ( create_log_field('localtime', '', false); # placeholder - 7/Nov/2006 - KBB create_database_field('date_time', 0, 0); create_database_field('day_of_week', 0, 0); create_database_field('hour_of_day', 0, 0); ); # if date else if (fieldname eq "time") then ( create_database_field('date_time', 0, 0); create_database_field('day_of_week', 0, 0); create_database_field('hour_of_day', 0, 0); ); # if time # Create derived fields for agent else if (fieldname eq "cs_user_agent") then ( # create_database_field('operating_system', 0, 0); # create_database_field('web_browser', 0, 0); # create_database_field('spider', 0, 0); ); # Create database field cs_ip and derived field for client IP else if (fieldname eq "c_ip") then ( create_database_field('c_ip', 0, 0); # create_database_field('location', 0, 0); ); # Create database field cs_referer and derived fields for referrer else if (fieldname eq "cs_referer") then ( create_database_field('cs_referer', 1, 9); # create_database_field('search_engine', 0, 0); # create_database_field('search_phrase', 0, 0); ); # Create derived file type field else if (fieldname eq "cs_uri_stem") then ( create_database_field('cs_uri_stem', 0, 9); # create_database_field('file_type', 0, 0); ); # Don't add a database field for numerical fields else if (subnode_exists(numerical_fields, fieldname)) then ( debug_message("Not adding numerical field: " . fieldname . "\n"); ); # Create a normal database field else create_database_field(fieldname, 0, 0); ); # if creating profile ); # while another field # Don't parse the #Fields line as a data line 'reject'; ); # if #Fields # Don't parse any other # lines as data lines else if (starts_with(current_log_line(), '#')) then ( 'reject'; ); ` log.parsing_filters.remove_time_milliseconds = ` if (matches_regular_expression(time, '^([0-9:]+)[.]')) then time = $1; ` # Log Filters log.filters = { empty_uri_query = { label = "$lang_admin.log_filters.empty_uri_query_label" comment = "$lang_admin.log_filters.empty_uri_query_comment" value = "if (cs_uri_query eq '-') then cs_uri_query = '(empty)';" disabled = false requires_fields = { cs_uri_query = true } } # empty_uri_query add_cs_uri_query = { label = "$lang_admin.log_filters.add_cs_uri_query_label" comment = "$lang_admin.log_filters.add_cs_uri_query_comment" value = "cs_uri_stem = cs_uri_stem . '?' . cs_uri_query" disabled = false requires_fields = { cs_uri_query = true cs_uri_stem = true } } # add_cs_uri_query } # log.filters database.numerical_fields = { time_taken = { type = "int" integer_bits = 64 display_format_type = duration_milliseconds } # time_taken # time_taken_avg = { # label = "$lang_stats.field_labels.average $lang_stats.field_labels.time_taken" # default = false # log_field = "time_taken" # requires_log_field = true # type = "int" # integer_bits = 64 # aggregation_method = "average" # average_denominator_field = "hits" # display_format_type = duration_milliseconds # } # time_taken } # database.numerical_fields create_profile_wizard_options = { # How the reports should be grouped in the report menu report_groups = { date_time_group = "" content_group = { cs_uri_stem = true cs_uri_query = true } visitor_demographics_group = { c_ip = true cs_username = true } referrer_group = { cs_referer = true } server_group = { s_sitename = true s_computername = true s_ip = true s_port = true cs_host = true } other_group = { cs_version = true # spider = true cs_method = true sc_status = true sc_substatus = true sc_win32_status = true cs_cookie = true } } # report_groups # This plug-in does its derived fields manually--don't automatically create operating_system, web_browser, etc. manual_derived_fields = true snapons = { # Attach a web_server_package snapon web_server_package = { snapon = "web_server_package" name = "web_server_package" label = "$lang_admin.snapons.web_server_package.label" parameters = { user_agent_field.parameter_value = "cs_user_agent" page_field.parameter_value = "cs_uri_stem" client_ip_field.parameter_value = "c_ip" server_response_field.parameter_value = "sc_status" referrer_field.parameter_value = "cs_referer" authenticated_user_field.parameter_value = "cs_username" hits_field = { parameter_value = "$lang_stats.field_labels.hits" final_node_name = "hits" } # hits visitors_field = { parameter_value = "$lang_stats.field_labels.unique_client_ips" final_node_name = "unique_client_ips" } # visitors_field sc_bytes_field = { parameter_value = "$lang_stats.field_labels.sc_bytes" final_node_name = "sc_bytes" } # sc_bytes_field cs_bytes_field = { parameter_value = "$lang_stats.field_labels.cs_bytes" final_node_name = "cs_bytes" } # cs_bytes_field } # parameters } # web_server_package } # snapons } # create_profile_wizard_options } # iis_advanced_logging_module