ironport_sseries_w3c_pattern = { plugin_version = "1.0.1" info.1.manfacturer = "Cisco" info.1.device = "IronPort Web Services Appliance (WSA S-Series) (pseudo-W3C with pattern header)" info.1.version.1 = "" # 2014-02-12 - 1.0 - GMF - Initial implementation. # 2014-02-12 - 1.0.1 - GMF - Changed autodetection to use #Software line # The name of the log format log.format.format_label = "Cisco IronPort Web Services Appliance (WSA S-Series) (pseudo-W3C with pattern header)" log.miscellaneous.log_data_type = "generic_w3c" log.miscellaneous.log_format_type = "firewall" # The log is in this format if any of the first ten lines match this regular expression log.format.autodetect_regular_expression = `#Software: AsyncOS for Web` log.format.date_format = "seconds_since_jan1_1970" log.format.time_format = "seconds_since_jan1_1970" # This handles #Fields lines, and creates log and database fields from them log.filter_preprocessor = ` if (matches_regular_expression(current_log_line(), '^#Fields: (.*)$')) then ( string fields = $1; string fieldname; v.logfieldindex = 1; string numerical_fields = "profiles." . internal.profile_name . ".database.numerical_fields"; # Send the line to all parsing servers distribute_format_line(current_log_line()); # Reset all fields to index 0, so any previous values they had won't carry over if we get new values here node log_fields = "profiles"{internal.profile_name}{"log"}{"fields"}; node log_field; foreach log_field log_fields ( @log_field{"index"} = 0; @log_field{"subindex"} = 0; ); # This subroutine creates a database field subroutine(create_database_field(string fieldname), ( #echo("create_database_field: " . fieldname); # echo("create_database_field(" . fieldname . ")"); string databasefieldpath = "profiles." . internal.profile_name . ".database.fields." . fieldname; (databasefieldpath . "") = ""; node databasefield = databasefieldpath; # set_subnode_value(databasefield, "label", fieldname); databasefield; )); subroutine(create_log_field(string fieldname, string type, bool withindex), ( # echo("create_log_field(" . fieldname . "; type=" . type . ")"); string logfieldpath = "profiles." . internal.profile_name . ".log.fields." . fieldname; (logfieldpath . "") = ""; node logfield = logfieldpath; # set_subnode_value(logfield, "label", fieldname); if (withindex) then ( set_subnode_value(logfield, "index", v.logfieldindex); #echo("Created log field with " . logfield . " with index=" . v.logfieldindex); v.logfieldindex++; ); set_subnode_value(logfield, "subindex", 0); if (type ne '') then set_subnode_value(logfield, "type", type); logfield; )); # Extract the fields on at a time # 2012-07-24 - Added support for tabs in Fields line, e.g, /pub/logs/Examples/wowza_media_server_pro/wowzamediaserver_stats.log.2012-03-08.2012-03-08.txt while (matches_regular_expression(fields, '^(%[^ ]+)[ ](.*)$') or matches_regular_expression(fields, '^(%[^ ]+)$')) ( string patternfield = $1; fields = $2; # Chop off digit in e.g. %2r if (matches_regular_expression(patternfield, '^%[0-9]+(.*)')) then patternfield = '%' . $1; string unconverted_fieldname = "unknown"; if (false) then ( ) else if (patternfield eq "%B") then unconverted_fieldname = "bytes"; else if (patternfield eq "%a") then unconverted_fieldname = "c-ip"; else if (patternfield eq "%F") then unconverted_fieldname = "c-port"; else if (patternfield eq "%M") then unconverted_fieldname = "CMF"; else if (patternfield eq "%C") then unconverted_fieldname = "cs(Cookie)"; else if (patternfield eq "%Server:") then unconverted_fieldname = "sc(Server)"; else if (patternfield eq "%b") then unconverted_fieldname = "sc-body-size"; else if (patternfield eq "%s") then unconverted_fieldname = "sc-bytes"; else if (patternfield eq "%h") then unconverted_fieldname = "sc-http-status"; else if (patternfield eq "%w") then unconverted_fieldname = "sc-result-code"; else if (patternfield eq "%W") then unconverted_fieldname = "sc-result-code-denial"; else if (patternfield eq "%V") then unconverted_fieldname = "time"; # else if (patternfield eq "%t") then unconverted_fieldname = "timestamp"; else if (patternfield eq "%t") then unconverted_fieldname = "date_time"; # Use Sawmill-style field name for timestamp else if (patternfield eq "%l") then unconverted_fieldname = "user-type"; else if (patternfield eq "%D") then unconverted_fieldname = "x-acltag"; else if (patternfield eq "%XO") then unconverted_fieldname = "x-avc-app"; else if (patternfield eq "%Xb") then unconverted_fieldname = "x-avc-behavior"; else if (patternfield eq "%XH") then unconverted_fieldname = "x-avc-reqbody-scanverdict"; else if (patternfield eq "%XN") then unconverted_fieldname = "x-avc-reqbody-scanverdict"; else if (patternfield eq "%XG") then unconverted_fieldname = "x-avc-reqhead-scanverdict"; else if (patternfield eq "%XM") then unconverted_fieldname = "x-avc-resphead-scanverdict"; else if (patternfield eq "%Xu") then unconverted_fieldname = "x-avc-type"; else if (patternfield eq "%XB") then unconverted_fieldname = "x-avg-bw"; else if (patternfield eq "%XT") then unconverted_fieldname = "x-bw-throttled"; else if (patternfield eq "%e") then unconverted_fieldname = "x-elapsed-time"; else if (patternfield eq "%E") then unconverted_fieldname = "x-error-code"; else if (patternfield eq "%H/%d") then unconverted_fieldname = "x-hierarchy-origin"; else if (patternfield eq "%i") then unconverted_fieldname = "x-icap-server"; else if (patternfield eq "%Xp") then unconverted_fieldname = "x-icap-verdict"; else if (patternfield eq "%Xl") then unconverted_fieldname = "x-ids-verdict"; else if (patternfield eq "%x") then unconverted_fieldname = "x-latency"; else if (patternfield eq "%L") then unconverted_fieldname = "x-local_time"; else if (patternfield eq "%Xg") then unconverted_fieldname = "x-mcafee-av-detecttype"; else if (patternfield eq "%Xf") then unconverted_fieldname = "x-mcafee-av-scanerror"; else if (patternfield eq "%Xh") then unconverted_fieldname = "x-mcafee-av-virustype"; else if (patternfield eq "%Xe") then unconverted_fieldname = "x-mcafee-filename"; else if (patternfield eq "%Xd") then unconverted_fieldname = "x-mcafee-scanverdict"; else if (patternfield eq "%Xj") then unconverted_fieldname = "x-mcafee-virus-name"; else if (patternfield eq "%X2") then unconverted_fieldname = "x-req-dvs-scanverdict"; else if (patternfield eq "%X4") then unconverted_fieldname = "x-req-dvs-threat-name"; else if (patternfield eq "%X3") then unconverted_fieldname = "x-req-dvs-verdictname"; else if (patternfield eq "%XS") then unconverted_fieldname = "x-request-rewrite"; else if (patternfield eq "%X0") then unconverted_fieldname = "x-resp-dvs-scanverdict"; else if (patternfield eq "%X1") then unconverted_fieldname = "x-resp-dvs-threat-name"; else if (patternfield eq "%XZ") then unconverted_fieldname = "x-resp-dvs-verdictname"; else if (patternfield eq "%Xr") then unconverted_fieldname = "x-result-code"; else if (patternfield eq "%w/%h") then unconverted_fieldname = "x-resultcode-httpstatus"; else if (patternfield eq "%Xy") then unconverted_fieldname = "x-sophos-file-name"; else if (patternfield eq "%Xx") then unconverted_fieldname = "x-sophos-scanerror"; else if (patternfield eq "%Xz") then unconverted_fieldname = "x-sophos-virus-name"; else if (patternfield eq "%?BLOCK_SUSPECT_USER_AGENT,MONITOR_SUSPECT_USER_AGENT?%1") then unconverted_fieldname = "x-s2p-first-byte-time"; else if (patternfield eq "%:1<") then unconverted_fieldname = "x-c2p-first-byte-time"; else if (patternfield eq "%:1>") then unconverted_fieldname = "x-p2c-first-byte-time"; else if (patternfield eq "%:a") then unconverted_fieldname = "x-p2p-auth-svc-time"; else if (patternfield eq "%:A<") then unconverted_fieldname = "x-p2p-avc-svc-time"; else if (patternfield eq "%:A>") then unconverted_fieldname = "x-p2p-avc-wait-time"; else if (patternfield eq "%:b") then unconverted_fieldname = "x-s2p-body-time"; else if (patternfield eq "%:b<") then unconverted_fieldname = "x-c2p-body-time"; else if (patternfield eq "%:b>") then unconverted_fieldname = "x-p2c-body-time"; else if (patternfield eq "%:>c") then unconverted_fieldname = "x-p2p-fetch-time"; else if (patternfield eq "%:C>") then unconverted_fieldname = "x-p2p-dca-resp-wait-time"; else if (patternfield eq "%:C<") then unconverted_fieldname = "x-p2p-dca-resp-svc-time"; else if (patternfield eq "%:d") then unconverted_fieldname = "x-p2p-dns-svc-time"; else if (patternfield eq "%:h") then unconverted_fieldname = "x-s2p-header-time"; else if (patternfield eq "%:h<") then unconverted_fieldname = "x-c2p-header-time"; else if (patternfield eq "%:h>") then unconverted_fieldname = "x-s2p-header-time"; else if (patternfield eq "%:m<") then unconverted_fieldname = "x-p2p-mcafee-resp-svc-time"; else if (patternfield eq "%:m>") then unconverted_fieldname = "x-p2p-mcafee-resp-wait-time"; else if (patternfield eq "%:") then unconverted_fieldname = "x-p2p-sophos-resp-svc-time"; else if (patternfield eq "%:") then unconverted_fieldname = "x-p2p-sophos-resp-wait-time"; else if (patternfield eq "%:r") then unconverted_fieldname = "x-p2p-reputation-svc-time"; else if (patternfield eq "%:s") then unconverted_fieldname = "x-p2p-asw-req-svc-time"; else if (patternfield eq "%:w<") then unconverted_fieldname = "x-p2p-webroot-resp-svc-time"; else if (patternfield eq "%:w>") then unconverted_fieldname = "x-p2p-webroot-resp-wait-time"; #echo("patternfield=" . patternfield . "; unconverted_fieldname=" . unconverted_fieldname); # Clean up the field name fieldname = ''; for (int i = 0; i < length(unconverted_fieldname); i++) ( string c = lowercase(substr(unconverted_fieldname, i, 1)); if (!matches_regular_expression(c, '^[a-z0-9]$')) then c = '_'; fieldname .= c; ); while (matches_regular_expression(fieldname, '^(.*)_$')) fieldname = $1; #echo("fieldname: " . fieldname); # Get the log field type string log_field_type = ''; # Create the log field #echo("Creating log field: " . fieldname); create_log_field(fieldname, log_field_type, true); # If we're creating a profile, create the database fields too. if (node_exists("volatile.creating_profile")) then ( #echo("creating database field; fieldname=" . fieldname); if (false) then ( ) # Handle date by creating date_time and derived database fields else if (fieldname eq "timestamp") then ( create_database_field('date_time'); create_database_field('day_of_week'); create_database_field('hour_of_day'); ); # if date # Don't add a database field for numerical fields else if (numerical_fields?{fieldname}) then ( debug_message("Not adding numerical field: " . fieldname . "\n"); ); # Create a normal database field else create_database_field(fieldname); ); # if creating profile ); # while another field # Don't parse the #Fields line as a data line 'reject'; ); # if #Fields # Don't parse any other # lines as data lines else if (starts_with(current_log_line(), '#')) then ( 'reject'; ); ` # Log Filters log.filters = { set_page_for_worm = { label = "$lang_admin.log_filters.set_page_for_worm_label" comment = "$lang_admin.log_filters.set_page_for_worm_comment" value = "if (starts_with(worm, '(')) then '' else cs_uri_stem = '(worm)';" requires_fields = { worm = true cs_uri_stem = true } } # set_page_for_worm # This filter tacks the page parameters ("URL query") onto the end of the page field empty_uri_query = { label = "$lang_admin.log_filters.empty_uri_query_label" comment = "$lang_admin.log_filters.empty_uri_query_comment" value = "if (cs_uri_query eq '-') then cs_uri_query = '(empty)';" disabled = true requires_fields = { cs_uri_query = true } } # empty_uri_query mark_entry = { label = '$lang_admin.log_filters.mark_entry_label' comment = '$lang_admin.log_filters.mark_entry_comment' value = 'accesses = 1;' } # mark_entry } # log.filters database.numerical_fields = { accesses = { requires_log_field = false default = true entries_field = true } # accesses sc_bytes = { integer_bits = 64 display_format_type = "bandwidth" } x_elapsed_time = { type = "float" display_format_type = duration_milliseconds } # x_elapsed_time } # database.numerical_fields create_profile_wizard_options = { # How the reports should be grouped in the report menu report_groups = { date_time_group = "" content_group = { cs_url = true cs_mime_type = true } # content_group source_group = { c_ip = true location = true domain_description = true organization = true domain = true isp = true } # source_group other_group = { x_resultcode_httpstatus = true cs_method = true cs_username = true x_hierarchy_origin = true x_acltag = true x_result_code = true x_suspect_user_agent = true } # other_group } # report_groups snapons = { # Attach a top_level_domain snapon top_level_domain = { snapon = "top_level_domain" name = "top_level_domain" label = "$lang_admin.snapons.top_level_domain.label" parameters = { url_field.parameter_value = "cs_url" field_name = { parameter_value = "$lang_admin.field_labels.top_level_domain" final_node_name = "top_level_domain" } } # parameters } # top_level_domain # Attach a File Type snapon file_type = { snapon = "file_type" name = "file_type" label = "$lang_admin.snapons.file_type.label" parameters = { url_field.parameter_value = "cs_url" file_type_field_name = { parameter_value = "$lang_admin.field_labels.file_type" final_node_name = "file_type" } } # parameters } # file_type # Attach a page_views field page_views = { snapon = "page_views" name = "page_views" label = "$lang_admin.snapons.page_views.label" prompt_to_attach = true prompt_to_attach_default = true parameters = { file_type_field.parameter_value = "file_type" server_response_field.parameter_value = "x_result_code" page_field.parameter_value = "cs_url" page_views_field_name.final_node_name = "page_views" page_views_field_name.parameter_value = "{=capitalize(lang_stats.field_labels.page_views)=}" } # parameters requires_log_fields = { cs_url = true x_result_code = true } } # page_views # Attach a gateway_reports snapon gateway_reports = { snapon = "gateway_reports" name = "gateway_reports" label = "$lang_admin.snapons.gateway_reports.label" parameters = { user_field.parameter_value = "c_ip" have_category_field.parameter_value = false # category_field.parameter_value = "category" host_field.parameter_value = "top_level_domain" page_views_field.parameter_value = "page_views" bytes_in_field.parameter_value = "size" sort_by_field.parameter_value = "page_views" } # parameters } # gateway_reports } # snapons } # create_profile_wizard_options } # ironport_sseries_w3c_pattern