blue_coat_w3_c = { # The name of the log format log.format.format_label = "Blue Coat W3C Log Format (ELFF)" log.miscellaneous.log_data_type = generic_w3c log.miscellaneous.log_format_type = "web_server" log.format.server_type = firewall # The log is in this format if any of the first ten lines match this regular expression log.format.autodetect_regular_expression = "^#Software: SGOS [234]\." # Literal apostrophes can appear in field values, and should not be treated as quotes log.format.treat_apostrophes_as_quotes = false # The format of dates and times in this log log.format.date_format = yyyy-mm-dd log.format.time_format = hh:mm:ss statistics.miscellaneous.entry_name = request # Don't track these fields as discrete database fields auto_setup.omit_database_fields = sc_bytes,cs_bytes,duration,time_taken,s_sitename log.fields = { url = { type = "page" } } # Log Filters log.filters = { set_url = { label = "Set URL" value = "url = cs_host;" comment = "Set the hostname as the URL" disabled = true requires_fields = { url = true cs_host = true cs_uri_scheme = false # cs_badvalue1 = true } } # set_url set_url_two = { label = "Set URL" value = "url = cs_uri_scheme . '://' . cs_host;" comment = "Append scheme to the host, set as URL" requires_fields = { cs_host = true url = true cs_uri_scheme = true # cs_badvalue2 = true } } # set_url_two set_url3 = { label = "Set URL" value = "url = cs_uri_stem;" comment = "Use cs_uri_stem to set the URL. Note that the domain may not be included in the string for transparent proxies." requires_fields = { url = true cs_uri_stem = true # cs_host = false # cs_uri_path = false } } # set_url3 set_url_from_uri = { label = "Set URL from cs_uri" value = " if (url eq '(empty)') then url = cs_uri; if (matches_regular_expression(url, '^([a-z]+://[^/]+/)')) then url = $1; " comment = "Use the cs_uri field to get the URL; and chop off everything after the hostname to keep the database simple" requires_fields = { cs_uri = true url = true } } # log_full_url log_full_url = { disabled = true label = "Track Full URL" value = "if (cs_uri_path ne '-') then url = url . cs_uri_path;" comment = "Include full path information in the URL" requires_fields = { cs_uri_path = true url = true } } # log_full_url active_content_url = { disabled = false label = "Active Content URLs" value = "if ((file_type eq 'CAB') or (file_type eq 'OCX') or (file_type eq 'ZIP') or (file_type eq 'EXE') or (file_type eq 'ISO') or (file_type eq 'COM') or (file_type eq 'DLL') or (file_type eq 'RAR')) then url = url . cs_uri_path;" comment = "Include full path information in the URL for downloads and active content" requires_fields = { cs_uri_path = true url = true file_type = true } } # active_content_url nobytes_on_deny = { disabled = false label = "nobytes_on_deny" value = "if (s_action eq 'TCP_DENIED') then sc_bytes = 0" comment = "This filter drops bytes for denied requests so that they don't affect overall bandwidth statistics" requires_fields = { s_action = true sc_bytes = true } } # nobytes_on_deny strip_query = { disabled = true label = "Strip Query" value = "if (starts_with(cs_uri_query, '?')) then cs_uri_query = substr(cs_uri_query, 1);" comment = "Strip off leading ? from query" requires_fields = { cs_uri_query = true } } # strip_query include_query = { disabled = true label = "Append Query" value = "if (cs_uri_query ne '-') then url = url . '?' . cs_uri_query;" comment = "Appends query string to the url" requires_fields = { cs_uri_query = true url = true } } # include_query # This filter replaces url parameters (page.html?param1+param2+...) with ?(parameters), to simplify the url hierarchy simplify_query = { disabled = true label = "simplify_query" value = "if (matches_regular_expression(url, '^(.*\\?).*\\$')) then url = '\\$1(parameters)';" comment = "replace query parameters to simplify the URL hierarchy" requires_fields = { url = true cs_uri_stem = true } } # simplify_query detect_page_views = { label = '$lang_admin.log_filters.detect_page_views_label' comment = '$lang_admin.log_filters.detect_page_views_comment' value = "if ((file_type eq 'JPEG') or (file_type eq 'JPG') or (file_type eq 'GIF') or (file_type eq 'ICO') or (file_type eq 'PNG') or (file_type eq 'CSS') or (file_type eq 'SWF') or (file_type eq 'JS')) then page_views = 0; else page_views = 1;" requires_fields = { file_type = true page_views = true rs_content_type = false } } # detect_page_views detect_page_views2 = { label = '$lang_admin.log_filters.detect_page_views_label' comment = '$lang_admin.log_filters.detect_page_views_comment' value = "if starts_with(rs_content_type, 'text/html') then page_views = 1; else page_views = 0;" requires_fields = { page_views = true rs_content_type = true } } # detect_page_views2 # This filter strips off the filename for non-page-views, to improve performance strip_non_page_views = { disabled = true label = '$lang_admin.log_filters.strip_non_page_views_label' comment = '$lang_admin.log_filters.strip_non_page_views_comment' value = "if (page_views == 0) then url = substr(url, 0, last_index(url, '/') + 1) . '(nonpage)';" requires_fields = { page_views = true url = true } } # strip_non_page_views # Mark this as a hit mark_as_hit = { label = "mark_as_hit" comment = "marks this request as a hit" value = "hits = 1;" requires_fields = { hits = true } } # mark_as_hit } # log.filters # Always include a "url" field in the log and database log.fields.url = { type = "page" label = "URL" index = 999 subindex = 999 hierarchy_dividers = "" left_to_right_hierarchy = "true" leading_divider = "false" case_sensitive = "false" } # url log.field_options = { sessions_page_field = "url" sessions_visitor_id_field = "c_ip" sessions_event_field = "page_views" } database.fields.url = { label = "url" type = "string" log_field = "url" suppress_top = "1" suppress_bottom = "3" always_include_leaves = "false" } # url # File type database field database.fields.file_type = { label = "file type" log_field = "file_type" type = "string" suppress_top = 0 suppress_bottom = 2 } # file_type # Force there to be host and date_time fields in advance, so we get prompted for day-by-day and bottom-level log.fields.c_ip = { type = "host" label = "$lang_stats.field_labels.c_ip" index = "1" subindex = "0" hierarchy_dividers = "." left_to_right_hierarchy = "false" leading_divider = "false" case_sensitive = "false" } # c_ip log.fields.date_time = { label = "$lang_stats.field_labels.date_time" type = "date_time" derived_from_1 = "date" derived_from_2 = "time" index = "0" subindex = "0" } # date_time # add a worm database field database.fields.worm = { label = "worm" log_field = "worm" type = "string" suppress_top = 0 suppress_bottom = 2 } # worm database.numerical_fields = { hits = { label = $lang_stats.field_labels.hits default = false requires_log_field = false type = int display_format_type = integer entries_field = true } # hits page_views = { label = $lang_stats.field_labels.page_views default = true requires_log_field = false type = int display_format_type = integer } # page_views visitors = { label = $lang_stats.field_labels.visitors default = false requires_log_field = true log_field = c_ip type = unique display_format_type = integer } # visitors cs_bytes = { label = $lang_stats.field_labels.cs_bytes default = false requires_log_field = true type = float display_format_type = bandwidth } # cs_bytes sc_bytes = { label = $lang_stats.field_labels.sc_bytes default = false requires_log_field = true type = float display_format_type = bandwidth } # sc_bytes # removing time_taken from the default plugin, as this numerical option confuses many users # time_taken = { # label = $lang_stats.field_labels.time_taken # default = false # requires_log_field = true # type = int # display_format_type = duration_compact # } # time_taken } # database.numerical_fields create_profile_wizard_options = { date_time_tracking = true host_tracking = true ######### START OF FINAL_STEP CODE final_step = ` include "templates.admin.profiles.setup_reports_util"; string profile = "profiles." . volatile.new_profile_name; # Remove non-essential database fields to reduce size and complexity # avoids creating reports for these items delete_database_field(profile, 'cs_uri_query'); delete_database_field(profile, 's_hierarchy'); delete_database_field(profile, 's_supplier_name'); delete_database_field(profile, 'location'); delete_database_field(profile, 'spider'); delete_database_field(profile, 'domain_description'); delete_database_field(profile, 'cs_uri_path'); delete_database_field(profile, 'cs_uri_stem'); delete_database_field(profile, 'cs_uri'); delete_database_field(profile, 's_ip'); delete_database_field(profile, 'cs_content_type'); # Start with the standard reports based on remaining DB fields add_standard_reports(profile); # Set sessions defaults (profile . ".statistics.miscellaneous.session_timeout") = "240"; (profile . ".statistics.miscellaneous.remove_reloads_from_sessions") = "false"; # enable date column on log detail report (profile . ".statistics.reports.log_detail.report_elements.log_detail.columns.date_time.visible") = "true"; node dbfield = profile . ".database.fields"; node reports = profile . ".statistics.reports"; string reportname = "none"; # create a category variable - set it to the category type in use with this log format string category = "none"; if (node_exists(dbfield . ".sc_filter_category")) then category = "sc_filter_category"; else if (node_exists(dbfield . ".cs_category")) then category = "cs_category"; # category related reports and report options if (category ne 'none') then ( # create a user-action-category xrefgroup add_field_to_xref_group(profile, "xrefgroup1", "date_time"); add_field_to_xref_group(profile, "xrefgroup1", "page_views"); add_field_to_xref_group(profile, "xrefgroup1", "cs_username"); add_field_to_xref_group(profile, "xrefgroup1", category); add_field_to_xref_group(profile, "xrefgroup1", "sc_filter_result"); add_field_to_xref_group(profile, "xrefgroup1", "sc_bytes"); add_field_to_xref_group(profile, "xrefgroup1", "cs_bytes"); add_field_to_xref_group(profile, "xrefgroup1", "visitors"); add_field_to_xref_group(profile, "xrefgroup1", "time_taken"); add_field_to_xref_group(profile, "xrefgroup1", "hits"); # create a domain to category xref group add_field_to_xref_group(profile, "domain_category", "date_time"); add_field_to_xref_group(profile, "domain_category", "cs_host"); add_field_to_xref_group(profile, "domain_category", "cs_username"); add_field_to_xref_group(profile, "domain_category", category); add_field_to_xref_group(profile, "domain_category", "page_views"); add_field_to_xref_group(profile, "domain_category", "sc_bytes"); add_field_to_xref_group(profile, "domain_category", "cs_bytes"); add_field_to_xref_group(profile, "domain_category", "visitors"); add_field_to_xref_group(profile, "domain_category", "time_taken"); add_field_to_xref_group(profile, "domain_category", "hits"); # Add a Pie Chart to the Content Categories Report # set to page views by default, change to sc_bytes if it exists node scfgraph = profile . ".statistics.reports." . category . ".report_elements." . category; (scfgraph . ".show_graph") = true; (scfgraph . ".graph.pie_chart") = true; (scfgraph . ".graph.image_width") = 300; (scfgraph . ".graph.image_height") = 300; if node_exists(dbfield . ".sc_bytes") then (scfgraph . ".graph.numerical_fields.sc_bytes") = true; else if node_exists(dbfield . ".page_views") then (scfgraph . ".graph.numerical_fields.page_views") = true; # Add multi-column reports including category info if (node_exists(dbfield . ".cs_username")) then ( add_table_report_element(profile, "category_by_user", "Categories by User", "category_by_user", "Categories by User", "cs_username", category, "page_views", "page_views", "descending", false); add_table_report_element(profile, "users_by_category", "Users by Category", "users_by_category", "Users by Category", category, "cs_username", "page_views", "page_views", "descending", false); reportname = ".category_by_user."; (reports . reportname . "report_elements" . reportname . "default_report_on_zoom") = "cs_host"; string reportname = ".cs_username."; (reports . reportname . "report_elements" . reportname . "default_report_on_zoom") = "domains_by_category"; reportname = ".users_by_category."; (reports . reportname . "report_elements" . reportname . "default_report_on_zoom") = "cs_host"; # Filter unauthenticated requests from user_by_category report # Disabled until inverse cross-reference table queries are supported #(profile . ".statistics.reports.users_by_category.filter.expression") = "not (cs_username within '-')"; ); if (node_exists(dbfield . ".c_ip")) then ( add_table_report_element(profile, "category_by_ip", "Categories by IP", "category_by_ip", "Categories by IP", "c_ip", category, "page_views", "page_views", "descending", false); reportname = ".c_ip."; (reports . reportname . "report_elements" . reportname . "default_report_on_zoom") = "domains_by_category"; reportname = ".category_by_ip."; (reports . reportname . "report_elements" . reportname . "default_report_on_zoom") = "cs_host"; ); if (node_exists(dbfield . ".sc_filter_result")) then ( add_table_report_element(profile, "result_by_category", "Category Actions", "result_by_category", "Category Actions", category, "sc_filter_result", "page_views", "page_views", "descending", false); ); if (node_exists(dbfield . ".cs_host")) then ( add_table_report_element(profile, "domains_by_category", "Domains by Category", "domains_by_category", "Domains by Category", category, "cs_host", "page_views", "page_views", "descending", false); reportname = ".individual_sessions."; (reports . reportname . "report_elements" . reportname . "default_report_on_zoom") = "domains_by_category"; ); ); # category related report options # add c_ip to the worm xref group if (node_exists(dbfield . ".worm")) then ( add_field_to_xref_group(profile, "worm", "c_ip"); add_table_report_element(profile, "ip_by_worm", "Worms", "ip_by_worm", "Worms", "worm", "c_ip", "hits", "hits", "descending", false); ); # add c_ip to the worm xref group # Add multi-column reports # -------------------------------------------- # Add domain to IP and username if (node_exists(dbfield . ".cs_host")) then ( reportname = ".s_action."; (reports . reportname . "report_elements" . reportname . "default_report_on_zoom") = "cs_host"; if (node_exists(dbfield . ".cs_username")) then ( add_table_report_element(profile, "domains_by_user", "Domains by User", "domains_by_user", "Domains by User", "cs_username", "cs_host", "page_views", "page_views", "descending", false); add_table_report_element(profile, "users_by_domain", "Users by Domain", "users_by_domain", "Users by Domain", "cs_host", "cs_username", "page_views", "page_views", "descending", false); reportname = "." . category . "."; (reports . reportname . "report_elements" . reportname . "default_report_on_zoom") = "domains_by_user"; reportname = ".result_by_category."; (reports . reportname . "report_elements" . reportname . "default_report_on_zoom") = "domains_by_user"; reportname = ".cs_host."; (reports . reportname . "report_elements" . reportname . "default_report_on_zoom") = "cs_username"; reportname = ".domains_by_user."; (reports . reportname . "report_elements" . reportname . "default_report_on_zoom") = "log_detail"; reportname = ".users_by_domain."; (reports . reportname . "report_elements" . reportname . "default_report_on_zoom") = "log_detail"; reportname = ".rs_content_type."; (reports . reportname . "report_elements" . reportname . "default_report_on_zoom") = "domains_by_user"; ); if (node_exists(dbfield . ".c_ip")) then ( add_table_report_element(profile, "domains_by_ip", "Domains by IP", "domains_by_ip", "Domains by IP", "c_ip", "cs_host", "page_views", "page_views", "descending", false); add_table_report_element(profile, "urls_by_ip", "URLs by IP", "urls_by_ip", "URLs by IP", "c_ip", "url", "page_views", "page_views", "descending", false); add_table_report_element(profile, "ip_by_domain", "IPs by Domain", "ip_by_domain", "IPs by Domain", "cs_host", "c_ip", "page_views", "page_views", "descending", false); reportname = ".domains_by_ip."; (reports . reportname . "report_elements" . reportname . "default_report_on_zoom") = "log_detail"; ); ); # Add ICAP Reports if (node_exists(dbfield . ".x_virus_id")) then ( if (node_exists(dbfield . ".c_ip")) then ( add_table_report_element(profile, "virus_by_ip", "ICAP Virus IP Detail", "virus_by_ip", "ICAP Virus IP Detail", "c_ip", "x_virus_id", "page_views", "page_views", "descending", false); reportname = ".x_virus_id."; (reports . reportname . "report_elements" . reportname . "default_report_on_zoom") = "c_ip"; ); if (node_exists(dbfield . ".url")) then ( add_table_report_element(profile, "virus_with_url", "ICAP Virus URL detail", "virus_with_url", "ICAP Virus URL detail", "x_virus_id", "url", "page_views", "page_views", "descending", false); ); if (node_exists(dbfield . ".cs_username")) then ( add_table_report_element(profile, "virus_by_user", "ICAP Virus User Detail", "virus_by_user", "ICAP Virus User Detail", "cs_username", "x_virus_id", "page_views", "page_views", "descending", false); ); # filter the null value '-' from the x_virus_id reports # disabled until inverse cross-reference table queries are supported #(profile . ".statistics.reports.x_virus_id.report_elements.x_virus_id.filter.expression") = "not (x_virus_id within '-')"; #(profile . ".statistics.reports.virus_by_ip.report_elements.virus_by_ip.filter.expression") = "not (x_virus_id within '-')"; #(profile . ".statistics.reports.virus_with_url.report_elements.virus_with_url.filter.expression") = "not (x_virus_id within '-')"; #(profile . ".statistics.reports.virus_by_user.report_elements.virus_by_user.filter.expression") = "not (x_virus_id within '-')"; ); # Create a report displaying authentication failures # --------------------------------------------------- if (node_exists(dbfield . ".cs_username")) and (node_exists(dbfield . ".sc_status")) then ( add_table_report_element(profile, "user_auth_fail", "Authentication Failures", "user_auth_fail", "Authentication Failures", "cs_username", "c_ip", "hits", "hits", "descending", false); # Tailor the filters for the user_auth_fail report (profile . ".statistics.reports.user_auth_fail.filter.expression") = "((not(cs_username within '-')) and ((sc_status within 407) or (sc_status within 401)))"; (profile . ".statistics.reports.user_auth_fail.report_elements.user_auth_fail.columns.0.header_label") = "Failed Usernames"; (profile . ".statistics.reports.user_auth_fail.report_elements.user_auth_fail.sub_table.show_totals_row") = false; reportname = ".user_auth_fail."; (reports . reportname . "report_elements" . reportname . "default_report_on_zoom") = "log_detail"; ); # Create group based reports and (xrefs currently commented out) if the cs_groups log field exists if (node_exists(dbfield . ".cs_auth_group")) then ( if (node_exists(dbfield . "." . category)) then add_table_report_element(profile, "category_by_group", "Categories by Group", "category_by_group", "Categories by Group", "cs_auth_group", category, "page_views", "page_views", "descending", false); if (node_exists(dbfield . ".cs_host")) then add_table_report_element(profile, "domain_by_group", "Domains by Group", "domain_by_group", "Domains by Group", "cs_auth_group", "cs_host", "page_views", "page_views", "descending", false); if (node_exists(dbfield . ".cs_username")) then add_table_report_element(profile, "user_by_group", "Users by Group", "user_by_group", "Users by Group", "cs_auth_group", "cs_username", "page_views", "page_views", "descending", false); #add_field_to_xref_group(profile, "group_user_xref", "date_time"); #add_field_to_xref_group(profile, "group_user_xref", "cs_auth_groups"); #add_field_to_xref_group(profile, "group_user_xref", "cs_username"); #add_field_to_xref_group(profile, "group_user_xref", category); #add_field_to_xref_group(profile, "group_user_xref", "page_views"); #add_field_to_xref_group(profile, "group_user_xref", "sc_bytes"); #add_field_to_xref_group(profile, "group_user_xref", "cs_bytes"); #add_field_to_xref_group(profile, "group_user_xref", "visitors"); #add_field_to_xref_group(profile, "group_user_xref", "time_taken"); #add_field_to_xref_group(profile, "group_user_xref", "hits"); add_field_to_xref_group(profile, "cs_auth_group", category); ); # Set Default zoom for hour of day report, cs_username settings reportname = ".hour_of_day."; (reports . reportname . "report_elements" . reportname . "default_report_on_zoom") = "cs_username"; # Filter unauthenticated requests from username reports # Disabled until inverse cross-reference table queries are supported #(profile . ".statistics.reports.cs_username.filter.expression") = "not (cs_username within '-')"; #(profile . ".statistics.reports.users_by_domain.filter.expression") = "not (cs_username within '-')"; #Classify uri-stem or uri-path as page instead of URL if (node_exists(profile . ".log.fields.cs_uri_stem")) then ( (profile . ".log.fields.url.type") = "flat"; (profile . ".log.fields.cs_uri_stem.type") = "page"; ) else if (node_exists(profile . ".log.fields.cs_uri_path")) then ( (profile . ".log.fields.url.type") = "flat"; (profile . ".log.fields.cs_uri_path.type") = "page"; ); ` ######### END OF FINAL_STEP CODE report_groups = { overview = true traffic_group = { date_time = true days = true day_of_week = true hour_of_day = true } security_group = { spider = false x_virus_id = true virus_by_ip = true virus_by_user = true virus_with_url = true worm = false ip_by_worm = true } filtering_group = { category = true cs_category = true cs_categories = true sc_filter_category = true result_by_category = true category_by_user = true category_by_ip = true sc_filter_result = false domains_by_category = true } content_group = { cs_host = true domains_by_user = true domains_by_ip = true urls_by_ip = true url = true file_type = true s_sitename = false s_computername = false s_ip = false s_supplier_name = false s_hierarchy = false cs_uri_path = true rs_content_type = true } users_group = { user_auth_fail = true c_ip = true ip_by_domain = true users_by_domain = true users_by_category = true domain_description = false location = false cs_username = true } technical_group = { cs_method = true cs_version = true sc_status = true s_port = false r_port = false s_action = true cs_uri_scheme = false cs_content_type = true web_browser = true operating_system = true } department_group = { cs_auth_group = true category_by_group = true user_by_group = true domain_by_group = true } sessions_group = { sessions_overview = true session_paths = false session_page_paths = false entry_pages = false exit_pages = false session_pages = true session_users = true individual_sessions = true } log_detail = true } # report_groups } # create_profile_wizard_options not_supported = { } # not_supported } # blue_coat_w3_c