Tonight I will discuss some more on function pointer to symbol translation using displacement from base.  because we found that such translation seems difficult without SymInitialize method that takes the handle of the target process. We will look at getting that information from dumps. MSDN says that the handle passed to SymInitialize must be the same value passed to all other symbol handlers functions called by the process. It is the handle that the functions use to identify the caller and locate the correct symbol information. Only that a process that acts like a debugger should not call the GetCurrentProcess to provide the handle and instead provide the handle of the target process. Thus the handle merely seems to be a token for a context. The other parameters to this function are UserSearchPath and fInvadeProcess. The UserSearchPath are the series of paths to be searched for a symbol file. This is an optional parameter. The fInvadeProcess when enumerates the loaded module for the process.
When you try out the SymInitialize with a bogus non-zero handle and follow it with a SymFromAddr call, it returns the error indicating that the handle is invalid.
Now we could use the

DWORD64 WINAPI SymLoadModuleEx(
  _In_  HANDLE hProcess,
  _In_  HANDLE hFile,
  _In_  PCTSTR ImageName,
  _In_  PCTSTR ModuleName,
  _In_  DWORD64 BaseOfDll,
  _In_  DWORD DllSize,
  _In_  PMODLOAD_DATA Data,
  _In_  DWORD Flags
);

but that method also doesn't help us specify the handle of a non-existent process.
So the only option left is to try creating a process with CreateProcess and DEBUG_PROCESS flag to specify that the created process is in debugging mode.

BOOL WINAPI CreateProcess(
  _In_opt_     LPCTSTR lpApplicationName,
  _Inout_opt_  LPTSTR lpCommandLine,
  _In_opt_     LPSECURITY_ATTRIBUTES lpProcessAttributes,
  _In_opt_     LPSECURITY_ATTRIBUTES lpThreadAttributes,
  _In_         BOOL bInheritHandles,
  _In_         DWORD dwCreationFlags, <------------------------------------- DEBUG_PROCESS
  _In_opt_     LPVOID lpEnvironment,
  _In_opt_     LPCTSTR lpCurrentDirectory,
  _In_         LPSTARTUPINFO lpStartupInfo,
  _Out_        LPPROCESS_INFORMATION lpProcessInformation
);

So I tried out with :

// Raw2Sym.cpp : Defines the entry point for the console application.

//

#include "stdafx.h"

#include "windows.h"

#include "dbghelp.h"

#pragma comment(lib, "dbghelp.lib")

int _tmain(int argc, _TCHAR* argv[])

{

//HMODULE hDbgModule = LoadLibrary(L"dbghelp.dll");

if (argc < 3) 

{

printf("Usage: raw2sym base_addr func_ptr func_ptr ...\r\n");

return -1;

}

DWORD  error = 0;

HANDLE hProcess = 0;

bool initialized = false;

hProcess = GetCurrentProcess();

printf ("hProcess = %ld\r\n", hProcess);

_TCHAR wszPath[MAX_PATH+1] = {0};

WCHAR  szImageName[MAX_PATH] = TEXT("foo.exe");

DWORD64 dwBaseAddr = _tcstoui64(argv[1], NULL, 16);

printf("Base Address: %p\r\n", dwBaseAddr);

HANDLE hExe = 0;

HANDLE hFile = 0;

if (!GetCurrentDirectory(MAX_PATH, wszPath))

{

error = GetLastError();

printf("GetCurrentDirectory returned error: %d\r\n", error);

goto exit;

}

else

{

wprintf(L"Current directory is %s\r\n", wszPath);

}

STARTUPINFO si;

        PROCESS_INFORMATION pi;

        ZeroMemory( &si, sizeof(si) );

        si.cb = sizeof(si);

        ZeroMemory( &pi, sizeof(pi) );

        if(!CreateProcess(L"foo.exe", NULL, 

    NULL, NULL, false,DEBUG_PROCESS, NULL, wszPath, &si, &pi))

{

error = GetLastError();

printf("CreateProcess returned error: %d\r\n", error);

goto exit;

}

hExe = pi.hProcess;

if(!hExe)

{

printf("raw2sym must have splunkd exe and pdb in the same folder as the tool.\r\n");

return -1;

}

else

{

printf ("hExe = 0x%x\r\n", hExe);

}

if (!SymInitializeW(hExe, wszPath, FALSE))

{

// SymInitialize failed

error = GetLastError();

printf("SymInitialize returned error : %d\r\n", error);

goto exit;

}

else

{

initialized = true;

}

        SymSetOptions(SYMOPT_UNDNAME | SYMOPT_DEFERRED_LOADS);

hFile = CreateFile(L"foo.pdb", 0, 0, NULL, 0, 0, NULL);

DWORD64 loadedAddress = SymLoadModuleEx(hExe, hFile, "foo.pdb", NULL,dwBaseAddr , 0, NULL, SLMFLAG_VIRTUAL);

if(loadedAddress == 0)

{

// SymLoadModuleEx failed

error = GetLastError();

printf("SymLoadModuleEx returned error : %d\n", error);

goto exit;

}

else

{

printf("Loaded Address is %p: \r\n", loadedAddress);

}

for (int i = 2 ; i < argc; i++)

{

DWORD64  dwDisplacement = 0;

DWORD64  dwAddress = _tcstoui64(argv[i], NULL, 16);

char buffer[sizeof(SYMBOL_INFO) + MAX_SYM_NAME * sizeof(TCHAR)];

PSYMBOL_INFO pSymbol = (PSYMBOL_INFO)buffer;

pSymbol->SizeOfStruct = sizeof(SYMBOL_INFO);

pSymbol->MaxNameLen = MAX_SYM_NAME;

if (!SymSetScopeFromAddr(hProcess, loadedAddress))

{

DWORD error = GetLastError();

printf("SymSetScopeFromAddr for %s returned error : %d\r\n", argv[i], error);

}

if (SymFromAddr(hProcess, dwAddress, &dwDisplacement, pSymbol))

{

printf("%s\r\n", &(pSymbol->Name));

}

else

{

// SymFromAddr failed

DWORD error = GetLastError();

printf("SymFromAddr for %s returned error : %d\r\n", argv[i], error);

}

}

exit:

if (initialized && hExe)

SymCleanup(hExe);

if (hExe)

CloseHandle(hExe);

if (hProcess)

CloseHandle(hProcess);

if (hFile)

CloseHandle(hFile);

return 0;

}

But that returns an error:

c:\downloads\Raw2Sym\x64\Release>Raw2Sym.exe 000007f79b1b0000 000007f79b1bbb90

hProcess = -1

Base Address: 000007F79B1B0000

Current directory is c:\downloads\Raw2Sym\x64\Release

hExe = 0x34

Loaded Address is 000007F79B1B0000:

SymSetScopeFromAddr for 0 returned error : 6

SymFromAddr for 0 returned error : 6

Error code: (Win32) 0x6 (6) - The handle is invalid.

Instead found a helpful resource here : http://www.debuginfo.com/examples/dbghelpexamples.html
and giving it a try. The code mentioned under SymLoadFromPdb.cpp gives some suggestions but doesn't work either.

c:\downloads\Raw2Sym\x64\Release>Raw2Sym.exe c:\downloads\Raw2Sym\x64\Release\splunkd.pdb 00

1b0000 000007f79b1bbb90

Loading symbols for: c:\downloads\Raw2Sym\x64\Release\splunkd.pdb ...

Error: SymLoadModule64() failed. Error code: 0

A better option might be to use DIA sdk instead as mentioned here : http://msdn.microsoft.com/en-us/library/4sy4ebfw.aspx .
 Conceptually, I don't know if any technique will allow this method since the process/dump needs to be available in memory or disk to associate the base address and the offset. The instruction set is unchanged after the compile so the pdb and exe will be the same. Resolving the symbols from the instruction doesn't vary
As Dia SDK shows here in the article querying the pdb: http://msdn.microsoft.com/en-us/library/eee38t3h.aspx

Software support folks in the field often encounter crash dumps at the customer's premise. There they have some dumps and they don't have any access to symbols particularly when the customer's operating system is Windows.What they see in the dumps or crash log reports are stack frames with raw addresses such as 0x7ff34555 and these are not yet resolved as module!class::method. To investigate the issue, they need to resolve the stackframes. To do this sometimes they have to open the crashdump with a debugger. But even the debugger needs the symbols to resolve the stack frame and the support folks can't take the symbols to the customer's machine. So there is a round-trip involved in taking the dumps to the developers because the symbols and the raw stacktrace are not together. Resolving raw function pointers to symbols in an offline manner is therefore desirable.
This can be achieved by using the same module that the debuggers use to translate the stack trace - dbghelp.dll. This module exports methods such as StackWalk and StackWalk64 that gives the stack trace when pointed to the dump. But more interestingly it has methods SymFromAddr and SymFromAddrW that just seems to do what we want. To use this method we call SymInitialize first so that we have a handle to the process. The method takes the following parameters :
- a handle retrieved from the SymInitialize method
- an Address pertaining to a frame in the stack for which the symbol should be located.
- a displacement from the beginning of a symbol or zero and this is optional
- a buffer to receive the SYMBOL_INFO resolved from the address.
If the function succeeds, it returns true. The SYMBOL_INFO struct has a member called MaxNameLen that should be set to the size of the name in characters. The name can be requested to be undecorated for easy readability.
The code from MSDN is something like this:

DWORD64  dwDisplacement = 0;
DWORD64  dwAddress = SOME_ADDRESS;

char buffer[sizeof(SYMBOL_INFO) + MAX_SYM_NAME * sizeof(TCHAR)];
PSYMBOL_INFO pSymbol = (PSYMBOL_INFO)buffer;

pSymbol->SizeOfStruct = sizeof(SYMBOL_INFO);
pSymbol->MaxNameLen = MAX_SYM_NAME;

if (SymFromAddr(hProcess, dwAddress, &dwDisplacement, pSymbol))
{
    // SymFromAddr returned success
   printf("%s\n", buffer);
}
else
{
    // SymFromAddr failed
    DWORD error = GetLastError();
    printf("SymFromAddr returned error : %d\n", error);
}

When we give some address it should be the relative offset from the base address.

In this post we will continue to look at logparser and Splunk. Consider a case when Splunk can translate searches to SQL queries. This would mean that all the operators that Splunk enables on the search bar such as regex, rex, top, stats, multikv, collect etc all work without seeing any difference between Splunk indexes or SQL data providers.  Splunk seamlessly parses the search operations on data without importing it into its indexes. In such cases there would have to be a translation of Splunk search operators into LINQ or SQL depending on where the data resides. A cursory look at the operators will suggest that the predicates be pushed down as close to the data as possible. In this case, Splunk keeps its index and data operators as close as possible. If the operators were to target the data on an external source there would be several copies of the data and translations involved. This will be similar to the pipe operation in Splunk. Splunk exposes several semantics that work well in pipe operations. This is very helpful to IT world for administration as well as for automation. What Splunk provides for analysis is significantly improved by its search language. While we can generalize the same operators for other data sources, the search language works well for Splunk data because of the fields extraction and event indexing.

LINQ on Splunk

Language-Integrated Query (LINQ) is a set of features introduced in Visual Studio 2008 that extends powerful query capabilities for use by applications. LINQ is widely used in web applications and services in a variety of industries such as finance, retail, telecommunication and insurance. LINQ provides two features. It enables constructs in the language that have become very popular with web developers. And it abstracts the underlying data source while enabling the same query logic to operate against any data source. The language syntax for querying are available as standard query operators and they are very much in nature to SQL ANSI standard which have an established foundation in data storage and retrieval. While SQL is ubiquitously used for relational data, the standard query operators are not restricted to work against a database. LINQ can be executed on XML, plaintext, CSV, databases etc. so long as the standard query operators can see a sequence of objects on which to perform their operations such as selection, conversion, merging etc. This enumeration of the items in a sequence is defined by an interface popularly known as IEnumerable.  The behavior demanded from collections implementing this interface is that the items can be extracted via iterations. At the same time, the data providers implement an interface called the IQueryable that can take an expression tree and execute it on the data source. Together these interfaces connect an application to the data source with the help of powerful querying capabilities.

Splunk also provides programmable interfaces to its data source via what are known as Entities and Collections, which provide a common mechanism for a plethora of resources. Most of the SDK makes REST API calls and hence operates on a single resource or a collection. Entity and EntityCollection both derive from common base class Resource.  Note that this pattern exposes the resources per se and not their querying capabilities.  When objects are instantiated they are local snapshots of values as read and copied from the server. Changes made on the server are not available until a refresh is called. Compare this with the object relational mapping (ORM) software that use LINQ patterns. The ORMs are able to interpret that the state of an object has become ‘dirty’ from local updates and needs to save to disk seamlessly.  At the same time, an observer pattern notifies the application of the updates made to the server data.

Splunk does have its own powerful querying capabilities.  Splunk search operators are similar to Unix style piped operators and are also available for use from the same SDK. These are exposed as searches and jobs and they can take a string containing the search query and execute it on the server. The results are then returned to the caller. Searches can be either one shot or real time.  The one-shot is a synchronous API and it returns a stream. Real-time searches return live events as they are indexed, and this type of search continues to arrive. To view the results from a real-time search, we view the preview results.  Some parameters to the Search Job enable this mode. In the normal execution mode, a Search Job is first created with a search string and then the job is polled for completion. A built-in XML parser can then render the stream of results.

Thus we note that there is a mismatch in the way the items are retrieved from the server from the server to the application between LINQ and Splunk search. That said, the power of Splunk can be made available via adapters via LINQ patterns.

LogParser and Splunk
We will cover some topics common to logParser and Splunk and see how we can integrate these.
LogParser is a versatile and powerful tool that enables searches over xml, csv etc. It has a universal query access to text based data.
LogParser can take a SQL expression on the command line and output the results that match the query.
Splunk has a unix style rich query operators to perform search that goes beyond just retrieving the results.
The advantage of a common LINQ expression for querying is that it can work with any data source. If we could consider logParser and Splunk as two different query based data providers, then arguably there is a way to support LINQ style querying over Splunk.
Let us on the other hand look at integrating LogParser and Splunk directly. While one can take the output of the other it is preferable that Splunk takes log parser as a modular input. More on this shortly.
Architecturally, there is considerable difference between LogParser and Splunk
Splunk to SQL connector apps already read relational data. There are plenty of apps that can connect to different SQL sources and perhaps use SQL queries.
However there is very limited apps that read logParser.
LogParser can read different kinds of data not just xml or csv. It can read different data such as event viewer.
The ability to query over such data is a major benefit contributing to its popularity on Windows systems.
Splunk could translate user queries to SQL and in this way have access to not only log parsers capabilities but also the data sources that are typically used with LogParser.

class Employee:
 def __init__(self, id, mgrId, name):
  self.id = id
  self.mgrId = mgrId
  self.name = name
    
# sample data
#E = [Employee(1, 0, 'ABC'), Employee(2, 1, 'DEF'), Employee(3, 1, 'GHI')]
E = [Employee(1, 1, 'ABC'), Employee(4,4,'IJK'), Employee(2, 1, 'DEF'), Employee(5,4,'MNO'), Employee(3, 1, 'GHI')]

def directReportsBFS(E,s):
 Q = []
 level = 0
 Q.append(s)
 while (len(Q) > 0):
  c = Q.pop(0)
  if c is None:
   level = level + 1
   continue
  Reports = [e for e in E if e.mgrId == c.id and e.mgrId != e.id]
  print 'Name:' + c.name + ' Level:' + str(level) + ' Reports:' +"[{0}]".format(", ".join(str(r.name) for r in Reports))
  Q.append(None)
  all(Q.append(report) for report in Reports)


[directReportsBFS(E,e) for e in E if e.mgrId == e.id]

In this blog post, I talk about matrix operations from the book on Algorithms and Data Structures. I'm covering these as part of the series of earlier posts from that book. The book says operations on matrices are at the heart of scientific computing. Efficient algorithms for working with matrices are therefore discussed. One important issue that arises in practice is numerical stability. Numerical stability means that there can be rounding errors which affect the results and are considered numerically unstable.
A matrix is a rectangular array of numbers. The elements of a matrix in row i and column j is aij. The set of all m * n matrices with real valued entries is denoted by R m*n.  The transpose of a matrix A is the matrix A' obtained by exchanging the rows and columns of A.
A vector is a one dimensional array of numbers. A column vector is a n * 1 matrix. A row vector is a 1 * n matrix. A transpose of a column vector is a row vector.
The unit vector is one whose ith element is 1 and all the other elements are 0. The size of a unit vector is clear from the context. A zero matrix is a matrix whose every entry is 0.
Square n * n matrices arise frequently. Several special cases of square matrices are of particular interest.
A diagonal matrix has aij = 0 whenever i != j. An upper triagonal matrix is one for which uij = 0 if i > j. All entries below the diagonal are zero.  A lower triagonal matrix is one for which lij = 0 for i < j.
Operations on matrices involve matrix addition such as in cij = aij + bij.
Matrix subtraction is the addition of a negative matrix.
In Matrix multiplication, we start with two matrices  A and B that are compatible in the sense that the number of columns of A is equal to the number of rows of B. We start with two matrices A and B that are compatible. then we perform C = AB where cik = Sum-j=1 to n (aij.bjk)
Matrices may have some of the associative properties.
Identity matrices are identities for matrix multiplication.
Matrix multiplication is associative
A(BC) = (AB)C
A(B+C) = AB + AC
(B+C)D = BD + CD
The inverse of an n x n matrix A is the n x n matrix A(-1) where paranthesis denotes superscript such that AA(-1) = A(-1)A = In
If a matrix has an inverse, it is called determinant or non-singular.
The vectors x1, x2, xn are linearly dependent if there exists co-efficients not all of which are zero.
The ijth minor of an n x n matrix A, for n > 1 is the (n-1)x(n-1) matrix obtained by deleting the ith row and the jth column of A.  The determinant of an n x n matrix A can be defined recursively in terms of its minors by
det (A) =   {  a11 if n = 1
                 { Sum for j = 1 to n [ (-1)^(1+j)   a1j   det(A 1,j) if n > 1
The term (-1)^(i+j).det(Aij) is known as the cofactor of the element aij.
The determinant of square matrix A has the following properties:
If any row or any column of A is zero, then det(A) is zero
The det(A) is multiplied by lambda if the entries of any one row or any one column are all multiplied by lambda.
The determinant of A is unchanged if the entries in one row are added to those in another row.
The det(A) equals the det(A transpose)
The det(A) is multiplied by -1 if any two rows or any two columns are exchanged.
For any square matrix, we have det(AB) = det(A).det(B)