# Copyright (c) 2010 Flowerfire, Inc. All Rights Reserved. palo_alto_networks_firewall_integrated = { plugin_version = "2.0.2" info.1.manufacturer = "Palo Alto Networks" info.1.device = "Firewall (Integrated Threat & Traffic)" info.1.version.1 = "1.3" info.1.version.2 = "2.0" info.1.version.3 = "3.0" info.1.version.4 = "3.1" info.1.version.5 = "4.0" # 2009-07-29 - GMF - 1.0 - Split this plug-in off from threat plug-in # 2009-09-09 - GMF - 1.0.1 - Added support for commas in URLs # 2010-04-13 - GMF - 1.0.2 - Added user database field (since it's used for sessions!) # 2010-10-05 - MSG - 1.0.2 - Edited info lines. # 2010-09-22 - KBB - 1.2 - Restored use of time_generated as the timestamp. Combined the two threat # variants, in autodetection and parsing. They are the same except for the 1,date in the front. Since # this is probably just truncation by the syslog, added same support for traffic logs. The first date # is not needed since time_generated is to be used per Palo Alto. Added log and database fields for the # bytes fields. Allowed for no placeholder quotes when there is no url in the THREAT line. # 2011-05-17 - GMF - 1.2.1 - Small performance optimizations: else if, and [^"]* in pathname. # 2010-12-08 - MSG - 1.3 - Changed suppress bottom value of page field to 9. # 2010-12-21 - MSG - 1.4 - Changed the log field name elapesed to elapsed. # 2011-01-25 - MSG & KBB - 1.4.1 - Fixed bug where matches_regular_expression reset positional variables # before they were used, causing threatid, category, severity and direction not to be set (THREAT). # 2011-02-11 - KBB - 1.4.2 - Changed support for missing 1,date to make space following those optional # to fix bug where some logs with those missing do not parse properly. # 2011-02-15 - KBB - 1.5 - Restored sessions by restoring the user database field. # 2011-09-06 - KBB - 1.6 - Added support for version 4. In version 4, certain fields which previously # contained values are now "FUTURE_USE". Since the values are still in the v4 logs, but designated # unpredictable by the v4 documentation, they are now suppressed for all versions. # 2011-11-18 - KBB - 1.6.1 - Restored log filters for removing parameters and non page views and # added one to simplify the url. They are all disabled by default. # 2011-07-14 - gas - 1.6.2 - Slight mod of the parsing regex to allow hostname and IPs in some fields. # (KBB - Somehow older code was checked in, with GAS change, so restored good code, then added GAS change. # I changed all instances of '([0-9.]+)' to '([a-z0-9.-]+)', whereas Graham did only some.) # 2012-02-08 - GMF - 2.0 - Disabled sequence_number. Added gateway reports. # 2012-03-21 - GMF - 2.0.1 - Removed start_time field (highly unique) # 2012-06-06 - GMF - 2.0.2 - Added requires_database_field page_views to gateways snapon # Here's an example line where the syslog received the message on June 27, and the received_time is June 27, # but time_generated is January 22! # This why I (GMF) have switched back to receive_time, pending more information on this from Palo Alto. # 2010-09-22 - KBB - Time must come from time_generated per Palo Alto. They have no explanation for # the anomalous date in this example, but say it isn't typical. # 2009-06-27 22:58:27 User.Info 1.2.3.4 Jun 27 23:05:52 1,06/27 23:05:52,0002A100287,THREAT,url,10,01/22 08:26:14,12.34.56.78,98.76.54.32,0.0.0.0,0.0.0.0,Domain Users Default,nt-something\someone,,web-browsing,vsys1,trust,untrust,ethernet1/2,ethernet1/1,Fwd_to_Kiwi_Sawmill,06/27 23:05:51,406938,1,1413,80,0,0,0x0,tcp,alert,googleads.g.doubleclick.net/pagead/ads?,(9999),search-engines,informational,0<000> # The name of the log format log.format.format_label = "Palo Alto Networks Firewall Integrated Log Format" log.miscellaneous.log_data_type = "syslog_required" log.miscellaneous.log_format_type = "firewall" # (Tabs in example changed to \t.) # The first date after the syslog header is being ignored because I'm not sure it is always there. KBB #2008-05-14 11:47:03\tUUCP.Info\t192.168.66.66\tMay 14 13:47:22 1,05/14 13:47:22,0001a100263,THREAT,url,3,12/31 18:09:46,192.168.66.66,12.34.56.78,0.0.0.0,0.0.0.0,rule1,,,gmail,vsys1,trust,untrust,ethernet1/2,ethernet1/1,Fwd_to_Syslogd,05/14 13:47:22,97,1,2222,80,0,0,0x0,tcp,alert,mail.google.com./mail/,(9999),web-based-e-mail,informational,0<000> #2008-08-25 16:53:36\tUser.Info\t192.168.101.10\tAug 26 14:51:07 1,08/26 14:51:07,0004A100238,THREAT,url,15,08/26 14:51:06,192.168.100.100,66.266.166.166,0.0.0.0,0.0.0.0,Outbound_Test_Nwk,,,web-browsing,vsys1,trust,untrust,ethernet1/2,ethernet1/1,threat to custom syslog,08/26 14:51:07,7364,1,50063,80,0,0,0x0,tcp,alert,safebrowsing.clients.google.com/safebrowsing/downloads?,(9999),unknown,informational,0<000> #2009-07-01 22:45:26 User.Info 10.222.2.222 Jul 1 22:45:26 1,2009/07/01 22:45:26,0002A100461,THREAT,url,5,2009/07/01 22:45:24,10.252.248.207,207.158.47.62,0.0.0.0,0.0.0.0,Less Restrictive,lion\tiger,,web-browsing,vsys1,inside,outside,ethernet1/2,ethernet1/1,Log_to_KIWI_Sawmill,2009/07/01 22:45:25,261303,1,2124,80,0,0,0x0,tcp,alert,"www.lionsandtigersandbearsohmy.com/errorpages/404.html",(9999),shopping,informational,0<000> #Jun 3 13:44:20 10.0.0.244 Jun 03 13: 44:20 1,06/03 13:44:20,0001a100200,TRAFFIC,start,8,06/03 13:44:19,10.0.10.10,16.166.16.66,0.0.0.0,0.0.0.0,mike,,,ntp,vsys1,l2-lan-trust,l2-lan-untrust,ethernet1/12,ethernet1/11,Forward to Mike,06/03 13:44:20,490821,1,123,123,0,0,0x0,udp,allow,90,90,90,1,06/03 13:44:20,0,any,0 # Another example, with different start-of-line format (no "1," and only one timestamp) [ThreadID:620871] #Jul 8 06:47:26 abc01-efgfw-01 06:47:26,0001A100332,THREAT,url,14,2009/07/08 06:46:53,12.34.56.78,98.76.54.32,23.45.67.89,87.65.43.21,User_Tracking,,,web-browsing,vsys25,assinet-inside,assinet-outside,ethernet1/2,ethernet1/10,ToSawMill,2009/07/08 06:47:26,981667,1,3557,80,3381,80,0x40,tcp,alert,"yahoo.com/",(9999),internet-portals,informational,0 # same, with layered syslogs #2010-06-22 13:17:50 Local7.Info 192.168.66.66 Jun 22 13:17:59 1,2010/06/22 13:17:59,0003C100949,TRAFFIC,end,117,2010/06/22 13:17:58,192.168.44.44,168.95.2.2,99.120.42.42,169.99.1.1,rule3,,,dns,vsys1,net.14-trust,net.13.14-untru,ethernet1/6,ethernet1/5,traffic-log,2010/06/22 13:17:58,141355,1,50878,53,33043,53,0x40,udp,allow,217,217,217,2,2010/06/22 13:17:27,1,any,0 log.format.autodetect_regular_expression = "[0-9]{2}:[0-9]{2}:[0-9]{2},[0-9A-Za-z]+,(TRAFFIC|THREAT),(url|virus|vulnerability|spyware|start|end)" log.format.autodetect_lines = 10000 log.format.parse_only_with_filters = "true" auto_setup.omit_database_fields = "start_time" # Log fields log.fields = { # receive_time = "" serial_number = "" type = "" sub_type = "" # config_version = "" source_ip = "" destination_ip = "" nat_source_ip = "" nat_destination_ip = "" rule_name = "" source_user = "" destination_user = "" application = "" virtual_system = "" source_zone = "" destination_zone = "" ingress_interface = "" egress_interface = "" log_forwarding_profile = "" # time_received = "" # session_id = "" source_port = "" destination_port = "" nat_source_port = "" nat_destination_port = "" flags = "" protocol = "" action = "" bytes = "" start_time = "" category = "" # repeat_count = "" # direction = "" sequence_number = "" action_flags = "" source_location = "" destination_location = "" # numeric bytes = "" # bytes_sent = "" # bytes_received = "" packets = "" elapsed_time = "" # threat only #miscellaneous = "" page.type = "page" # miscellaneous always contains a url # threat_id = "" content_type = "" # new in v4 user = "" # derived, for sessions # domain = "" # derived, from page } # log.fields # Log Filters log.filters = { remove_query = { label = "$lang_admin.log_filters.remove_query_label" comment = "$lang_admin.log_filters.remove_query_comment" value = "if (contains(page, '?')) then page = substr(page, 0, index(page, '?') + 1) . '(parameters)';" disabled = true } # remove_query # compute_domain = { # value = "if (matches_regular_expression(page, '^([^/]+)/')) then domain = $1;" # } # compute_domain detect_page_views = { label = '$lang_admin.log_filters.detect_page_views_label' comment = '$lang_admin.log_filters.detect_page_views_comment' value = "if ((sub_type ne 'url') or (category eq 'advertisements-and-popups') or (file_type eq 'JPEG') or (file_type eq 'JPG') or (file_type eq 'GIF') or (file_type eq 'ICO') or (file_type eq 'PNG') or (file_type eq 'CSS') or (file_type eq 'SWF') or (file_type eq 'JS')) then page_views = 0; else page_views = 1;" disabled = false } # detect_page_views simplify_url = { label = "$lang_admin.log_filters.simplify_url_label" comment = "$lang_admin.log_filters.simplify_url_comment" value = "if (matches_regular_expression(page, '^(([^:]+://|/)?[^/]+/)')) then page = $1 . '(omitted)'" disabled = false } # simplify_url strip_non_page_views = { label = '$lang_admin.log_filters.strip_non_page_views_label' comment = '$lang_admin.log_filters.strip_non_page_views_comment' value = "if (page_views == 0) then page = substr(page, 0, last_index(page, '/') + 1) . '(nonpage)';" disabled = true } # strip_non_page_views mark_entry = { label = '$lang_admin.log_filters.mark_entry_label' comment = '$lang_admin.log_filters.mark_entry_comment' value = 'events = 1;' } # mark_entry } # log.filters log.field_options = { sessions_page_field = "page" sessions_visitor_id_field = "user" sessions_event_field = "page_views" } # log.field_options log.parsing_filters.parse = ` # Get the year from the syslog date. v.syslog_date = get_collected_field('', 'date'); v.year = ''; if (matches_regular_expression(v.syslog_date, '([0-9]{4})')) then ( v.year = $1; ); v.session_user = ''; # Handle THREAT lines # v3 - These are the v3 field names. The corresponding v4 field names are now used for the database. ##Important fields: receive_time, subtype, src, dst, srcuser, dport, action, misc, category #Important fields: subtype, time_generated, src, dst, srcuser, dport, action, misc, category #All fields: domain,receive_time,serial,type,subtype,config_ver,time_generated,src,dst,natsrc,natdst,rule,srcuser,dstuser,app,vsys,from,to,inbound_if,outbound_if,logset,time_received,sessionid,repeatcnt,sport,dport,natsport,natdport,flags,proto,action,misc,threatid,category,severity,direction #2008-08-25 17:07:05\tUser.Info\t192.168.65.65\tAug 26 15:04:36 1,08/26 15:04:36,0004A100238,THREAT,url,15,08/26 15:04:35,192.168.65.66,206.206.236.66,0.0.0.0,0.0.0.0,Outbound_Test_Nwk,testuser,,web-browsing,vsys1,trust,untrust,ethernet1/2,ethernet1/1,threat to test syslog,08/26 15:04:36,8270,1,51502,80,0,0,0x0,tcp,alert,www.securityfocus.com/rss/vulnerabilities.xml,(9999),computing-and-internet,informational,0<000> #2008-05-14 11:47:56\tUUCP.Info\t192.168.66.66\tMay 14 13:48:14 1,05/14 13:48:14,0001a100263,THREAT,spyware,3,12/31 18:10:38,192.168.55.55,55.55.55.55,0.0.0.0,0.0.0.0,rule1,,,web-browsing,vsys1,trust,untrust,ethernet1/2,ethernet1/1,Fwd_to_Syslogd,05/14 13:48:14,125,1,2246,80,0,0,0x0,tcp,alert,d.yimg.com./us.yimg.com/i/us/p/cnn.com.web,(9999),unknown,informational,0<000> #2009-07-01 22:45:26 User.Info 10.222.2.222 Jul 1 22:45:26 1,2009/07/01 22:45:26,0002A100461,THREAT,url,5,2009/07/01 22:45:24,10.252.248.207,207.158.47.62,0.0.0.0,0.0.0.0,Less Restrictive,lion\tiger,,web-browsing,vsys1,inside,outside,ethernet1/2,ethernet1/1,Log_to_KIWI_Sawmill,2009/07/01 22:45:25,261303,1,2124,80,0,0,0x0,tcp,alert,"www.lionsandtigersandbearsohmy.com/errorpages/404.html",(9999),shopping,informational,0<000> # Jul 8 06:47:26 abcdef01-enetfw-02 06:47:26,0001A100332,THREAT,url,14,2009/07/08 06:46:53,10.12.34.56.78,98.76.54.32,23.45.67.89,87.65.43.21,User_Tracking,,,web-browsing,vsys25,assinet-inside,assinet-outside,ethernet1/2,ethernet1/10,ToSawMill,2009/07/08 06:47:26,981207,1,2334,80,40550,80,0x40,tcp,alert,"forums.somewhere.com/index.php?",(9999),unknown,informational,0 # No page, so no quotes #Jun 2 18:00:25 10.0.0.44 Jun 02 18: 00:25 1,06/02 18:00:25,0001a100200,THREAT,vulnerability,4,06/02 18:00:19,550.0.0.22,10.0.0.222,0.0.0.0,0.0.0.0,rule15,laughnetwork\libby,,msrpc,vsys1,l2-lan-trust,l2-lan-untrust,ethernet1/12,ethernet1/11,Forward to Mike,06/02 18:00:25,180718,1,2007,135,0,0,0x8000,tcp,alert,,Microsoft RPC Endpoint Mapper(30845),any,low,0 # v4 #Important fields: sub_type, generated_time, source_ip, destination_ip, source_user, destination_port, action, misc, category # All fields: FUTURE_USE, Receive Time, Serial Number, Type, Subtype, FUTURE_USE, Generated Time, Source IP, Destination IP, NAT Source IP, NAT Destination IP, Rule Name, Source User, Destination User, Application, Virtual System, Source Zone, Destination Zone, Ingress Interface, Egress Interface, Log Forwarding Profile, FUTURE_USE, Session ID, Repeat Count, Source Port, Destination Port, NAT Source Port, NAT Destination Port, Flags, Protocol, Action, Miscellaneous, Threat ID, Category, Severity, Direction, Sequence Number, Action Flags, Source Location, Destination Location, FUTURE_USE, Content Type #Aug 23 19:21:58 10.30.10.40 1,2011/08/23 19:16:48,0001C100768,THREAT,url,1,2011/08/23 19:16:47,172.16.11.111,172.16.22.122,0.0.0.0,0.0.0.0,tiger,,,web-browsing,vsys2,tiger,tiger,ethernet1/7,ethernet1/7,ubuntu_test_logs,2011/08/23 19:16:48,320859,1,32232,80,0,0,0x0,tcp,block-url,"www.tiger.com/RealMedia/ads/tiger.cgi/www.tiger.com/popup/@x02",(9999),spyware-and-adware,informational,client-to-server,0,0x0,172.16.0.0-172.31.255.255,172.16.0.0-172.31.255.255,0, if (matches_regular_expression(v.syslog_message, '(([^,]*),([0-9/]+) )? *([0-9]{2}:[0-9]{2}:[0-9]{2}),([^,]+),(THREAT),([^,]+),([^,]+),([0-9/]+) ([0-9]{2}:[0-9]{2}:[0-9]{2}),([a-z0-9.-]+),([a-z0-9.-]+),([a-z0-9.-]+),([a-z0-9.-]+),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([0-9/]+ [0-9]{2}:[0-9]{2}:[0-9]{2}),([^,]*),([0-9]*),([0-9]*),([0-9]*),([0-9]*),([0-9]*),([^,]*),([a-z]+),([^,]*),("[^"]*"|[^,]*),([^,]*),([^,]*),([^,]*),([^,]*)(,([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*))?$')) then ( v.repeat_count = $27; # v.original_repeat_count = $42; # Insert repeat_count copies of log line. if (v.repeat_count > 1) then ( v.in_context = "," . $25 . "," . $26 . "," . $27 . "," . $28 . "," . $29 . ","; v.once_only = "," . $25 . "," . $26 . "," . 1 . "," . $28 . "," . $29 . ","; v.line = replace_first(current_log_line(), v.in_context, v.once_only); for (int i = 0; i < v.repeat_count; i++) ( # set_subnode_value('volatile.log_line_insertions', i, v.line . "," . v.repeat_count); set_subnode_value('volatile.log_line_insertions', i, v.line); ); ); # Accept repeated and non-repeated lines. else ( v.user = $16; v.src = $11; v.date = $9; set_collected_field('', 'time', $10); # Commented fields are currently not needed and not specified in log.fields or database.fields. #set_collected_field('', 'domain', $2); #set_collected_field('', 'receive_time', $3 . " " . $4); set_collected_field('', 'serial_number', $5); set_collected_field('', 'type', $6); set_collected_field('', 'sub_type', $7); #set_collected_field('', 'config_version', $8); # $9 is date # $10 is time set_collected_field('', 'source_ip', $11); set_collected_field('', 'dst', $12); set_collected_field('', 'nat_source_ip', $13); set_collected_field('', 'nat_destination_ip', $14); set_collected_field('', 'rule_name', $15); set_collected_field('', 'source_user', $16); set_collected_field('', 'destination_user', $17); set_collected_field('', 'application', $18); set_collected_field('', 'virtual_system', $19); set_collected_field('', 'source_zone', $20); set_collected_field('', 'destination_zone', $21); set_collected_field('', 'ingress_interface', $22); set_collected_field('', 'egress_interface', $23); set_collected_field('', 'log_forwarding_profile', $24); #set_collected_field('', 'time_received', $25); #set_collected_field('', 'session_id', $26); # Don't store this. Varying line lengths break old method of keeping track of it. #set_collected_field('', 'repeat_count', $27); #if (v.original_repeat_count eq '') then ( # v.original_repeat_count = "1"; #); #set_collected_field('', 'repeat_count', v.original_repeat_count); set_collected_field('', 'source_port', $28); set_collected_field('', 'destination_port', $29); set_collected_field('', 'nat_source_port', $30); set_collected_field('', 'nat_destination_port', $31); set_collected_field('', 'flags', $32); set_collected_field('', 'protocol', $33); set_collected_field('', 'action', $34); #set_collected_field('', 'miscellaneous', $35); v.page = $35; set_collected_field('', 'threat_id', $36); set_collected_field('', 'category', $37); set_collected_field('', 'severity', $38); set_collected_field('', 'direction', $39); set_collected_field('', 'sequence_number', $41); set_collected_field('', 'action_flags', $42); set_collected_field('', 'source_location', $43); set_collected_field('', 'destination_location', $44); ##set_collected_field('', 'future_use', $45); #set_collected_field('', 'content_type', $46); if (matches_regular_expression(v.page, '^"(.*)"$')) then ( v.page = $1; ); set_collected_field('', 'page', v.page); if (matches_regular_expression(v.date, '^[0-9]{4}/[0-9]{2}/[0-9]{2}$')) then ( set_collected_field('', 'date', v.date); ); else if (v.year ne '') then ( set_collected_field('', 'date', v.year . "/" . v.date); ); else ( set_collected_field('', 'date', normalize_date(v.date, 'mm/dd')); ); v.session_user = v.src; if (v.user ne '') then ( v.session_user .= '_' . v.user; ); set_collected_field('', 'user', v.session_user); accept_collected_entry('', false); ); ); # if THREAT # Handle TRAFFIC lines # v3 - These are the v3 field names. The corresponding v4 field names are now used for the database. #Important fields: receive_time, sub_type, time_generated, src, dst, rule, srcuser, app, from, to, time_received, sessionid, dport, proto, action, bytes, bytes_sent, bytes_received, packets, start, elapsed, bytes, bytes_sent, bytes_received, packets, start, elapsed, category #All fields: domain, receive_time, serial, type, sub_type, config_ver, time_generated, src, dst, natsrc, natdst, rule, srcuser, dstuser, app, vsys, from, to, inbound_if, outbound_if, logset, time_received, sessionid, repeatcnt, sport, dport, natsport, natdport, flags, proto, action, bytes, bytes_sent, bytes_received, packets, start, elapsed, category, padding #Jun 3 13:44:20 10.0.0.244 Jun 03 13: 44:20 1,06/03 13:44:20,0001a100200,TRAFFIC,start,8,06/03 13:44:19,10.0.10.10,16.166.16.66,0.0.0.0,0.0.0.0,mike,,,ntp,vsys1,l2-lan-trust,l2-lan-untrust,ethernet1/12,ethernet1/11,Forward to Mike,06/03 13:44:20,490821,1,123,123,0,0,0x0,udp,allow,90,90,90,1,06/03 13:44:20,0,any,0 #2009-07-01 22:45:26 User.Info 10.222.2.222 Jul 1 22:45:26 1,2009/07/01 22:45:26,0002A100461,TRAFFIC,end,5,2009/07/01 22:45:24,22.122.2.122,10.222.2.122,0.0.0.0,0.0.0.0,Permit Any In,,,incomplete,vsys1,outside,inside,ethernet1/1,ethernet1/2,Log_to_KIWI_Sawmill,2009/07/01 22:45:25,313521,1,1247,81,0,0,0x0,tcp,allow,440,440,440,7,2009/07/01 22:44:52,30,any,0<000> #Sep 09 12:18:03 10.30.14.179 Sep 9 12:20:32 1,2010/09/09 12:20:32,0006C100489,TRAFFIC,end,49,2010/09/09 12:20:32,192.168.22.222,22.222.22.2,172.12.22.222,22.222.22.2,rule12,,,bittorrent,vsys1,L3-trust,L3-untrust-2,ethernet1/2,ethernet1/6,Panorama-229,2010/09/09 12:20:31,31959,1,11171,30638,2542,30638,0x42,udp,allow,464,464,464,2,2010/09/09 12:00:32,0,any,0 # v4 #All fields: FUTURE_USE, receive_time, serial_number, type, sub_type, FUTURE_USE, generated_time, source_ip, destination_ip, nat_source_ip, nat_destination_ip, rule_name, source_user, destination_user, application, virtual_system, source_zone, destination_zone, ingress_interface, egress_interface, log_forwarding_profile, FUTURE_USE, session_id, repeat_count, source_port, destination_port, nat_source_port, nat_destination_port, flags, protocol, action, bytes, FUTURE_USE, FUTURE_USE, packets, start_time, elapsed_time, category, FUTURE_USE, sequence_number, action_flags, source_location, destination_location, FUTURE_USE #Aug 23 20:38:25 10.20.30.40 1,2011/08/23 20:33:14,0001C100768,TRAFFIC,end,1,2011/08/23 20:33:13,172.16.2.222,172.16.3.133,0.0.0.0,0.0.0.0,testco policy,,,incomplete,vsys3,testco trust,testco trust,ethernet1/13,ethernet1/13,ubuntu_test_logs,2011/08/23 20:33:14,1353314,1,24410,45823,0,0,0x0,tcp,allow,198,198,198,4,2011/08/23 20:33:04,0,any,0,0,0x0,172.16.0.0-172.31.255.255,172.16.0.0-172.31.255.255,0 else if (matches_regular_expression(v.syslog_message, '(([^,]*),([0-9/]+) )? *([0-9]{2}:[0-9]{2}:[0-9]{2}),([^,]+),(TRAFFIC),([a-z]*),([0-9]*),([0-9/]+) ([0-9]{2}:[0-9]{2}:[0-9]{2}),([a-z0-9.-]+),([a-z0-9.-]+),([a-z0-9.-]+),([a-z0-9.-]+),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([^,]*),([0-9/]+ [0-9]{2}:[0-9]{2}:[0-9]{2}),([0-9]*),([0-9]*),([0-9]*),([0-9]*),([0-9]*),([0-9]*),([^,]*),([^,]*),([^,]*),([0-9]*),([0-9]*),([0-9]*),([0-9]*),([0-9/]+ [0-9]{2}:[0-9]{2}:[0-9]{2}),([0-9]*),([^,]*),([^,]*)(,([^,]*),([^,]*),([^,]*),([^,]*))?')) then ( v.repeat_count = $27; # v.original_repeat_count = $44; # Insert repeat_count copies of log line. if (v.repeat_count > 1) then ( v.in_context = "," . $25 . "," . $26 . "," . $27 . "," . $28 . "," . $29 . ","; v.once_only = "," . $25 . "," . $26 . "," . 1 . "," . $28 . "," . $29 . ","; v.line = replace_first(current_log_line(), v.in_context, v.once_only); for (int i = 0; i < v.repeat_count; i++) ( # set_subnode_value('volatile.log_line_insertions', i, v.line . "," . v.repeat_count); set_subnode_value('volatile.log_line_insertions', i, v.line); ); ); # Accept repeated and non-repeated lines. else ( # Commented fields are currently not needed and not specified in log.fields or database.fields. v.src = $11; v.date = $9; #set_collected_field('', 'domain', $2); # set_collected_field('', 'receive_time', $3 . " " . $4); set_collected_field('', 'serial_number', $5); # was serial set_collected_field('', 'type', $6); set_collected_field('', 'sub_type', $7); # set_collected_field('', 'config_version', $8); set_collected_field('', 'time', $10); set_collected_field('', 'source_ip', v.src); set_collected_field('', 'destination_ip', $12); set_collected_field('', 'nat_source_ip', $13); set_collected_field('', 'nat_destination_ip', $14); set_collected_field('', 'rule_name', $15); set_collected_field('', 'source_user', $16); set_collected_field('', 'destination_user', $17); set_collected_field('', 'application', $18); set_collected_field('', 'virtual_system', $19); set_collected_field('', 'source_zone', $20); set_collected_field('', 'destination_zone', $21); set_collected_field('', 'ingress_interface', $22); set_collected_field('', 'egress_interface', $23); set_collected_field('', 'log_forwarding_profile', $24); # set_collected_field('', 'time_received', $25); # set_collected_field('', 'session_id', $26); # Don't store repeat_count. Mechanism for remembering it breaks with varying line lengths. # set_collected_field('', 'repeat_count', $27); # if (v.original_repeat_count eq '') then ( # v.original_repeat_count = "1"; # ); # set_collected_field('', 'repeat_count', v.original_repeat_count); set_collected_field('', 'source_port', $28); set_collected_field('', 'destination_port', $29); set_collected_field('', 'nat_source_port', $30); set_collected_field('', 'nat_destination_port', $31); set_collected_field('', 'flags', $32); set_collected_field('', 'protocol', $33); set_collected_field('', 'action', $34); set_collected_field('', 'bytes', $35); # set_collected_field('', 'bytes_sent', $36); # set_collected_field('', 'bytes_received', $37); set_collected_field('', 'packets', $38); set_collected_field('', 'start_time', $39); set_collected_field('', 'elapsed_time', $40); set_collected_field('', 'category', $41); #set_collected_field('', 'padding', $42); set_collected_field('', 'sequence_number', $44); set_collected_field('', 'action_flags', $45); set_collected_field('', 'source_location', $46); set_collected_field('', 'destination_location', $47); if (matches_regular_expression(v.date, '^[0-9]{4}/[0-9]{2}/[0-9]{2}$')) then ( set_collected_field('', 'date', v.date); ); else if (v.year ne '') then ( set_collected_field('', 'date', v.year . "/" . v.date); ); else ( set_collected_field('', 'date', normalize_date(v.date, 'mm/dd')); ); accept_collected_entry('', false); ); ); # if TRAFFIC #else ( # debug # echo(v.syslog_message); #); ` # Database fields database.fields = { # receive_time = "" serial_number = "" type = "" sub_type = "" # config_version = "" source_ip = "" destination_ip = "" nat_source_ip = "" nat_destination_ip = "" rule_name = "" source_user = "" destination_user = "" application = "" virtual_system = "" source_zone = "" destination_zone = "" ingress_interface = "" egress_interface = "" log_forwarding_profile = "" # time_received = "" # session_id = "" source_port = "" destination_port = "" nat_source_port = "" nat_destination_port = "" flags = "" protocol = "" action = "" # 2012-03-21 - GMF - Removing highly unique field start_time # start_time = "" category = "" # repeat_count = "" # direction = "" # This is a *highly* unique field; disabling it by default # sequence_number = "" action_flags = "" source_location = "" destination_location = "" content_type = "" # threat only #miscellaneous = "" page = { suppress_bottom = 9 display_format_type = "page" } # page # threat_id = "" user = "" # derived, for sessions # domain = "" # derived, from page } # database.fields database.numerical_fields = { events = { default = true requires_log_field = false entries_field = true } # events page_views = { default = true requires_log_field = false } # page_views bytes = { type = "int" integer_bits = 64 display_format_type = "bandwidth" } # bytes # bytes_sent = { # type = "int" # integer_bits = 64 # display_format_type = "bandwidth" # } # bytes_sent # # bytes_received = { # type = "int" # integer_bits = 64 # display_format_type = "bandwidth" # } # bytes_received packets = { type = "int" integer_bits = 64 display_format_type = "bandwidth" } # packets elapsed_time = { type = "int" integer_bits = 64 display_format_type = "duration_compact" } # elapsed_time } # database.numerical_fields create_profile_wizard_options = { # How the reports should be grouped in the report menu report_groups = { date_time_group = "" source_group = { source_ip = true nat_source_ip = true source_user = true user = true source_port = true nat_source_port = true source_zone = true egress_interface = true source_location = true category_by_source_user = true page_by_source_user = true } # source_group destination_group = { destination_ip = true nat_destination_ip = true destination_user = true destination_port = true nat_destination_port = true destination_zone = true ingress_interface = true destination_location = true } # destination_group content_group = { type = true sub_type = true page = true file_type = true content_type = true category = true } # content_group other_group = { logging_device = true serial_number = true # config_version = true rule_name = true application = true virtual_system = true log_forwarding_profile = true # time_received = true # session_id = true flags = true protocol = true action = true start_time = true # repeat_count = true sequence_number = true action_flags = true } # other_group } # report_groups snapons = { # Attach a top_level_domain snapon top_level_domain = { snapon = "top_level_domain" name = "top_level_domain" label = "$lang_admin.snapons.top_level_domain.label" parameters = { url_field.parameter_value = "page" top_level_domain_field.parameter_value = "top_level_domain" top_level_domain_field_name = { parameter_value = "$lang_admin.field_labels.top_level_domain" final_node_name = "top_level_domain" } } # parameters } # top_level_domain # Attach a gateway_reports snapon gateway_reports = { snapon = "gateway_reports" name = "gateway_reports" label = "$lang_admin.snapons.gateway_reports.label" parameters = { user_field.parameter_value = "source_user" have_client_ip.parameter_value = false have_category_field.parameter_value = true category_field.parameter_value = "category" host_field.parameter_value = "top_level_domain" page_views_field.parameter_value = "page_views" have_bytes_in_field.parameter_value = true bytes_in_field.parameter_value = "bytes" have_bytes_out_field.parameter_value = false sort_by_field.parameter_value = "session_duration" } # parameters requires_database_fields = { page_views = true } } # gateway_reports # Add the standard reports add_standard_reports = { name = "add_standard_reports" label = "add_standard_reports" snapon = "add_standard_reports" } # add_standard_reports } # snapons } # create_profile_wizard_options } # palo_alto_networks_firewall_integrated