#
# log4j_parser snapon
#
# This snapon implements log4j parsing functionality, based on a specified PatternLayout directive.
#


log4j_parser = {

  version = "1.3.1"

  # 2012-02-08 - GMF - 1.1 - Fixed date format (%d) when not specified
  # 2012-02-10 - GMF - 1.2 - Added milliseconds_from_layout_construction (%r)
  # 2012-02-13 - GMF - 1.3 - Fixed support for right padded fields (e.g. %-5p)
  # 2012-02-16 - GMF - 1.3.1 - Fixed spaces in *_diagnostic_content fields.  Allowed %x to be empty.
  # 2012-02-24 - GMF - 1.3.2 - Added line breaks with <br> to the message field, and turned on skip_escaping for that field so it will render them.

  label = "$lang_admin.snapons.log4j_parser.label"
  comment = "$lang_admin.snapons.log4j_parser.comment"
  config_snapon_category = ""
  
  parameters = {
  
    pattern_layout = {
	
      parameter_value = ""
      validation_type = "string"
		
      form_element_label = "$lang_admin.snapons.log4j_parser.parameters.pattern_layout.form_element_label"
      form_element_type = "text"
      form_element_width = "500"
		
      description = ""
  	
    } # pattern_layout
    
  } # parameters
  
  
  parameters_form = {

    group_1 = {
      description = "$lang_admin.snapons.log4j_parser.parameters_form.group_1.description"
      parameters = {
        pattern_layout = true
      } # parameters
    } # group 1

  } # parameters_form


  attach_operations = {

    configure_parsing = {
      type = "execute_expression"
      expression = `

#echo("profile: " . profile);
#echo("profile: " . node_as_string(profile));
string pattern = @profile{"create_profile_wizard_info"}{"snapons"}{"parser"}{"parameters"}{"pattern_layout"}{"parameter_value"};
#echo("pattern: " . pattern);

  # This subroutine creates a database field
  subroutine(create_database_field(node profile, string fieldname), (
#echo("create_database_field: " . fieldname);
    node database_field = profile{"database"}{"fields"}{fieldname};

    if (fieldname eq 'message') then (
      @database_field{'sql_field_length'} = 10000;
      @database_field{'skip_escaping'} = true;
    );

  )); # create_database_field()

  # This subroutine creates an aggregating database field
  subroutine(create_aggregating_database_field(node profile, string fieldname, string type, string display_format_type), (
#echo("create_aggregating_database_field: " . fieldname);
    node database_field = profile{"database"}{"fields"}{fieldname};

    @database_field{"type"} = type;
#    @database_field{"integer_bits"} = 64;

#    if ('lang_stats.field_labels.tomcat_pattern'?{fieldname}) then
#      @database_field{"label"} = "{" . "=capitalize(expand(lang_stats.field_labels.tomcat_pattern." . fieldname . "))=" . "}";
#    else
      @database_field{"label"} = "{" . "=capitalize(expand(lang_stats.field_labels." . fieldname . "))=" . "}";
    @database_field{"display_format_type"} = display_format_type;

    @database_field{"aggregation_method"} = "sum";

  )); # create_database_field()

  subroutine(create_log_field(node profile, string fieldname, string type), (
#echo("create_log_field: fieldname=" . fieldname);
#echo("profile: " . profile);
    node logfield = profile{"log"}{"fields"}{fieldname};
#echo("logfield: " . logfield);
    @logfield{"type"} = type;
  )); ##=== create_log_field() ===##

  node log_fields_at_end = new_node();
  subroutine(create_log_field_at_end(node log_fields_at_end, string fieldname, string type), (
    log_fields_at_end{fieldname}{"type"} = type;
  ));


# Delete any existing log fields
delete_node(profile{"log"}{"fields"});

# Parse the string, building the regular expression from it, and the fields
string regexp = "^";
string field_regexp;
string timestamp_regexp;
string field_name;
string log_field_type;
string c;
string parsing_filter_set_collected_fields = "\n";
int fieldnum = 1;
bool final_field_is_m = false;
for (int i = 0; i < length(pattern); i++) (
#  echo("i: " . i);
  c = substr(pattern, i, 1);
#  echo("c: " . c);

  # Handle % variables
  if (c eq '%') then (
    i++;
    c = substr(pattern, i, 1);

    log_field_type = "flat";

    # Handle leading integer
    string leading_integer = "";
    bool right_padded = false;
    if (c eq '-') then
      right_padded = true;
    if (matches_regular_expression(c, '^[0-9-]$')) then (
      while (matches_regular_expression(c, '^[0-9-]$')) (
        leading_integer .= c;
        i++;
        c = substr(pattern, i, 1);
      ); # while integer
    ); # if integer
#echo("leading integer: " . leading_integer);

    # Remember if the field field is %m, possibly followed by %n.
    if (c eq 'm') then
      final_field_is_m = true;
    else if (c eq 'n') then
      ( );
    else
      final_field_is_m = false;

    if (false) then ( )

    else if (c eq 'c') then (
      field_regexp = '([^ ]+)';
      field_name = "logging_category";

      # Handle {} (by ignoring it)
      if (substr(pattern, i+1, 1) eq '{') then (
        i += 2;
        c = substr(pattern, i, 1);

        string curly_section = "";
        while ((c ne '}') and (i != length(pattern))) (
          curly_section .= c;
          i++;
          c = substr(pattern, i, 1);
        ); # while not }

      ); # if {}

    ); # c

    else if (c eq 'C') then (
      field_regexp = '([^ ]+)';
      field_name = "caller_class";
    ); # C

# Need to support format variants, at least the common ones.
    else if (c eq 'd') then (
      field_regexp = '([^ ]+)';
      field_name = "time";
      log_field_type = "time";

      # Handle {}
      if (substr(pattern, i+1, 1) eq '{') then (
        i += 2;
        c = substr(pattern, i, 1);

        string curly_section = "";
#echo("i: " . i);
#echo("length(pattern): " . length(pattern));
        while ((c ne '}') and (i != length(pattern))) (
#echo("  c=" . c);
          curly_section .= c;
          i++;
          c = substr(pattern, i, 1);
        ); # while not }

#echo("Curly section: {" . curly_section . "}");
      ); # if {}

      if ((curly_section eq "") or (curly_section eq "ISO8601") or (curly_section eq "yyyy-MM-dd HH:mm:ss,SSS")) then (
        timestamp_regexp = '([0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]) ([0-9][0-9]:[0-9][0-9]:[0-9][0-9]),[0-9][0-9][0-9]';
	create_log_field(profile, 'date', 'date');
	field_regexp = timestamp_regexp; 
      ); # ISO8601

      # Handle standard date format
      else if ((curly_section eq "DATE") or (curly_section eq "dd MMM yyyy HH:mm:ss,SSS")) then (
        timestamp_regexp = '([0-9][0-9] [A-Z][a-z][a-z] [0-9][0-9][0-9][0-9]) ([0-9][0-9]:[0-9][0-9]:[0-9][0-9]),[0-9][0-9][0-9]';
	create_log_field(profile, 'date', 'date');
	field_regexp = timestamp_regexp; 
      ); # DATE

      else if (curly_section eq "dd MMM yyyy HH:mm:ss:SSS") then (
        timestamp_regexp = '([0-9][0-9] [A-Z][a-z][a-z] [0-9][0-9][0-9][0-9]) ([0-9][0-9]:[0-9][0-9]:[0-9][0-9]):[0-9][0-9][0-9]';
	create_log_field(profile, 'date', 'date');
	field_regexp = timestamp_regexp;         
      ); # dd MMM yyyy HH:mm:ss:SSS

      # New date format from ThreadID:1285254 
      else if (curly_section eq "yyyy-MM-dd HH:mm:ss") then (
        timestamp_regexp = '([0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]) ([0-9][0-9]:[0-9][0-9]:[0-9][0-9])';
	create_log_field(profile, 'date', 'date');
	field_regexp = timestamp_regexp; 
      ); # yyyy-MM-dd HH:mm:ss

      else
        error("Unsupported date format %d{" . curly_section . "}.  Please contact Sawmill technical support if you need support for this format.  Or use %d{DATTE}"); #LM

    ); # d

    else if (c eq 'F') then (
      field_regexp = '([^ ]+)';
      field_name = "request_file_name";
    ); # F

    else if (c eq 'l') then (
      field_regexp = '([^ ]+)';
      field_name = "caller_location";
    ); # l

    else if (c eq 'L') then (
      field_regexp = '([^ ]+)';
      field_name = "request_line_number";
    ); # L

    else if (c eq 'm') then (
      field_regexp = '(.*)';
      field_name = "message";
    ); # m

    else if (c eq 'n') then (
      field_regexp = '';
      field_name = "line_separator";
    ); # n

    else if (c eq 'p') then (
      field_regexp = '([^ ]+)';
      field_name = "priority";
    ); # p

    else if (c eq 'm') then (
      field_regexp = '([^ ]+)';
      field_name = "time_elapsed";
    ); # m

    else if (c eq 'M') then (
      field_regexp = '([^ ]+)';
      field_name = "method_name";
    ); # M

    else if (c eq 'r') then (
      field_regexp = '([^ ]+)';
      field_name = "milliseconds_from_layout_construction";
    ); # r

    else if (c eq 't') then (
      field_regexp = '([^ ]+)';
      field_name = "thread_name";
    ); # t

    else if (c eq 'x') then (
      field_regexp = '([^ ]*)';
      field_name = "nested_diagnostic_content";
    ); # x

    else if (c eq 'X') then (
      field_regexp = '([^ ]+)';
      field_name = "mapped_diagnostic_content";
    ); # X

    else
      error("Invalid pattern; unknown % directive %" . c); # LM

    if (right_padded) then
      field_regexp .= " *";

#    echo("found field_name=" . field_name . "; field_regexp=" . field_regexp);

    # Don't create fields for line_separator
    if (field_name ne "line_separator") then (

      # Add this field to the regular expression
      regexp .= field_regexp;

      create_log_field(profile, field_name, log_field_type);
#    echo("NEED TO ADD FIELDS HERE (" . field_name . ")");

      if (field_name eq "time") then (
        create_log_field_at_end(log_fields_at_end, 'date_time', 'date_time');
        create_log_field_at_end(log_fields_at_end, 'day_of_week', 'flat');
        create_log_field_at_end(log_fields_at_end, 'hour_of_day', 'flat');
#echo("SCHEDULING date_time log field");
        create_database_field(profile, 'date_time');
        create_database_field(profile, 'day_of_week');
        create_database_field(profile, 'hour_of_day');
      );

      # Handle aggregating fields  
      else if (field_name eq "time_elapsed") then      
        create_aggregating_database_field(profile, field_name, "int", "duration_millis");

      # Handle aggregating fields  
      else if (field_name eq "milliseconds_from_layout_construction") then      
        create_aggregating_database_field(profile, field_name, "int", "duration_millis");

      # Create a normal database field
      else
        create_database_field(profile, field_name);

      if (field_name eq "time") then (
        parsing_filter_set_collected_fields .= "  set_collected_field('', 'date', $" . fieldnum . ");\n";
        fieldnum++;
      );
      parsing_filter_set_collected_fields .= "  set_collected_field('', '" . field_name . "', $" . fieldnum . ");\n";
      fieldnum++;


    ); # if not line_separator

  ); # if %

  # If it's not a variable, add it literally to the regular expression.
  else (

    if ((c eq '[') or (c eq ']') or (c eq '.') or (c eq '|') or (c eq '(') or (c eq ')')) then (
      regexp .= '\\\\';
    );

    regexp .= c;

  ); # if not %

); # for i

regexp .= "$";

# If the final field is %m, there might be a stack trace on multiple lines after the first line.  So, we need
# to use collect/accept to collect those into the message, and finally accept on the next YYYY-MM-DD HH-MM-S,mmm line.
#echo("Final regular expression: " . regexp);
if (final_field_is_m) then (
  string parsing_filter = "
# Accept each line on the following line
if (matches_regular_expression(current_log_line(), '" . timestamp_regexp . "')) then (
  set_collected_field('', 'message', replace_all(get_collected_field('', 'message'), '$', '_'));
  accept_collected_entry('', false);
);

# If this is a line with all the first fields, collect them
if (matches_regular_expression(current_log_line(), '" . regexp . "')) then (" .
parsing_filter_set_collected_fields .
");

# Otherwise, it must be a continuation of the previous message line; add it to the message field
else (
  set_collected_field('', 'message', get_collected_field('', 'message') . '<br />' . current_log_line());
);

";
  @profile{"log"}{"parsing_filters"}{"parse"} = parsing_filter;
  @profile{"log"}{"format"}{"parse_only_with_filters"} = true;

); # if final_field_is_m

# If the final field isn't %m, just use a parsing regular expression
else
  @profile{"log"}{"format"}{"parsing_regular_expression"} = regexp;

#echo("numerical fields: " . node_as_string(profile{"database"}{"numerical_fields"}));

# Create any final log fields
node lfae;
foreach lfae log_fields_at_end (
#echo("Final log field creation: " . node_name(lfae));
  create_log_field(profile, node_name(lfae), @lfae{"type"});
);
create_log_field(profile, "events", "flat");

#echo("Final log fields: " . node_as_string(profile{"log"}{"fields"}));
#echo("Final database fields: " . node_as_string(profile{"database"}{"fields"}));


`

    } # configure_parsing

  } # attach_operations

} # double_hits