In this post like in the previous, we will continue to look at Splunk integration with SQL and NoSQL systems. Specifically we will look at log parser and Splunk interaction. Splunk users know how to tran slate SQL queries to Splunk search queries. We use search operators for this. For non-Splunk users we could provide Splunk as a data store with log parser as a SQL interface. Therefore, we will look into providing Splunk searchable data as a COM input to log parser. A COM input simply implements a few methods for the log parser and abstracts the data store. These methods are :
OpenInput: Opens your data source and sets up any initial environment settings
GetFieldCount: returns the number of fields that the plugin provides
GetFieldName: returns the name of a specified field
GetFieldType : returns the datatype of a specified field
GetValue : returns the value of a specified field
ReadRecord : reads the next record from your data source
CloseInput: closes the data source and cleans up any environment settings
Together splunk and log parser brings the power of splunk to log parser users without requiring them to know about Splunk search commands. At the same time, they have the choice to search the Splunk indexes directly. The ability to use SQL makes Splunk more common and inviting to windows users.
<SCRIPTLET>
<registration
Description=“Splunk Input Log Parser Scriptlet"
Progid="Splunk.Input.LogParser.Scriptlet"
Classid="{fb947990-aa8c-4de5-8ff3-32a59fb66a6c}"
Version="1.00"
Remotable="False" />
<comment>
EXAMPLE: logparser "SELECT * FROM MAIN" -i:COM -iProgID:Splunk.Input.LogParser.Scriptlet
</comment>
<implements id="Automation" type="Automation">
<method name="OpenInput">
<parameter name="strValue"/>
</method>
<method name="GetFieldCount" />
<method name="GetFieldName">
<parameter name="intFieldIndex"/>
</method>
<method name="GetFieldType">
<parameter name="intFieldIndex"/>
</method>
<method name="ReadRecord" />
<method name="GetValue">
<parameter name="intFieldIndex"/>
</method>
<method name="CloseInput">
<parameter name="blnAbort"/>
</method>
</implements>
<SCRIPT LANGUAGE="VBScript">
Option Explicit
Const MAX_RECORDS = 5
Dim objAdminManager, objResultDictionary
Dim objResultsSection, objResultsCollection
Dim objResultElement
Dim objResultsElement, objResultElement
Dim intResultElementPos, intResult, intRecordIndex
Dim clsResult
Dim intRecordCount
' --------------------------------------------------------------------------------
' Open the input Result.
' --------------------------------------------------------------------------------
Public Function OpenInput(strValue)
intRecordIndex = -1
Set objResultDictionary = CreateObject("Scripting.Dictionary")
Set objResultsSection = GetSearchResults(“index=main”);
Set objResultsCollection = objResultsSection.Collection
If IsNumeric(strValue) Then
intResultElementPos = FindElement(objResultsCollection, "Result", Array("id", strValue))
Else
intResultElementPos = FindElement(objResultsCollection, "Result", Array("name", strValue))
End If
If intResultElementPos > -1 Then
Set objresultElement = objResultsCollection.Item(intResultElementPos)
Set objFtpServerElement = objResultElement.ChildElements.Item(“SearchResults”)
Set objResultsElement = objFtpServerElement.ChildElements.Item(“SearchResult).Collection
For intResult = 0 To CLng(objResultsElement.Count)-1
Set objResultElement = objResultsElement.Item(intResult)
Set clsResult = New Result
clsResult.Timestamp = objResultElement.GetPropertyByName(“timestamp”).Value
clsResult.Host = objResultElement.GetPropertyByName(“host”).Value
clsResult.Source = objResultElement.GetPropertyByName(“source”).Value
clsResult.SourceType = objResultElement.GetPropertyByName(“sourcetype”).Value
clsResult.Raw = objResultElement.GetPropertyByName(“raw”).Value
objResultDictionary.Add intResult,clsResult
Next
End If
End Function
' --------------------------------------------------------------------------------
' Close the input Result.
' --------------------------------------------------------------------------------
Public Function CloseInput(blnAbort)
intRecordIndex = -1
objResultDictionary.RemoveAll
End Function
' --------------------------------------------------------------------------------
' Return the count of fields.
' --------------------------------------------------------------------------------
Public Function GetFieldCount()
GetFieldCount = 5
End Function
' --------------------------------------------------------------------------------
' Return the specified field's name.
' --------------------------------------------------------------------------------
Public Function GetFieldName(intFieldIndex)
Select Case CInt(intFieldIndex)
Case 0:
GetFieldName = “Timestamp”
Case 1:
GetFieldName = “Host”
Case 2:
GetFieldName = “Source”
Case 3:
GetFieldName = “Sourcetype”
Case 4:
GetFieldName = “Raw”
Case Else
GetFieldName = Null
End Select
End Function
' --------------------------------------------------------------------------------
' Return the specified field's type.
' --------------------------------------------------------------------------------
Public Function GetFieldType(intFieldIndex)
' Define the field type constants.
Const TYPE_STRING = 1
Const TYPE_REAL = 2
Const TYPE_TIMESTAMP = 3
Const TYPE_NULL = 4
Select Case CInt(intFieldIndex)
Case 0:
GetFieldType = TYPE_TIMESTAMP
Case 1:
GetFieldType = TYPE_STRING
Case 2:
GetFieldType = TYPE_STRING
Case 3:
GetFieldType = TYPE_STRING
Case 4:
GetFieldType = TYPE_STRING
Case Else
GetFieldType = Null
End Select
End Function
' --------------------------------------------------------------------------------
' Return the specified field's value.
' --------------------------------------------------------------------------------
Public Function GetValue(intFieldIndex)
If objResultDictionary.Count > 0 Then
Select Case CInt(intFieldIndex)
Case 0:
GetValue = objResultDictionary(intRecordIndex).Timestamp
Case 1:
GetValue = objResultDictionary(intRecordIndex).Host
Case 2:
GetValue = objResultDictionary(intRecordIndex).Source
Case 3:
GetValue = objResultDictionary(intRecordIndex).SourceType
Case 4:
GetValue = objResultDictionary(intRecordIndex).Raw
Case Else
GetValue = Null
End Select
End If
End Function
' --------------------------------------------------------------------------------
' Read the next record, and return true or false if there is more data.
' --------------------------------------------------------------------------------
Public Function ReadRecord()
If objResultDictionary.Count > 0 Then
If intRecordIndex < (objResultDictionary.Count-1) Then
intRecordIndex = intRecordIdndex + 1
ReadRecord = True
Else
ReadRecord = False
End If
End If
End Function
Class Result
Public Timestamp
Public Host
Public Source
Public SourceType
Public Raw
End Class
</SCRIPT>
</SCRIPTLET>
Scriptlet Courtesy : Robert McMurray's blog
I will provide a class library for the COM callable wrapper to Splunk searchable data in C#.
The COM library that returns the search results can implement methods like this:
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Splunk;
using SplunkSDKHelper;
using System.Xml;
namespace SplunkComponent
{
[System.Runtime.InteropServices.ComVisible(false)]
public class SplunkComponent
{
public SplunkComponent()
{
// Load connection info for Splunk server in .splunkrc file.
var cli = Command.Splunk("search");
cli.AddRule("search", typeof(string), "search string");
cli.Parse(new string[] {"--search=\"index=main\""});
if (!cli.Opts.ContainsKey("search"))
{
System.Console.WriteLine("Search query string required, use --search=\"query\"");
Environment.Exit(1);
}
var service = Service.Connect(cli.Opts);
var jobs = service.GetJobs();
job = jobs.Create((string)cli.Opts["search"]);
while (!job.IsDone)
{
System.Threading.Thread.Sleep(1000);
}
}
[System.Runtime.InteropServices.ComVisible(false)]
public string GetAllResults()
{
var outArgs = new JobResultsArgs
{
OutputMode = JobResultsArgs.OutputModeEnum.Xml,
// Return all entries.
Count = 0
};
using (var stream = job.Results(outArgs))
{
var setting = new XmlReaderSettings
{
ConformanceLevel = ConformanceLevel.Fragment,
};
using (var rr = XmlReader.Create(stream, setting))
{
return rr.ReadOuterXml();
}
}
}
private Job job { get; set; }
}
}
https://github.com/ravibeta/csharpexamples/tree/master/SplunkComponent.