# -*- coding: utf-8 -*-
"""
CParser.py - C parsing library 
Copyright 2010  Luke Campagnola
Distributed under MIT/X11 license. See license.txt for more infomation.

Used for extracting data such as macro definitions, variables, typedefs, and function
signatures from C files (preferrably header files).
"""

import sys, re, os

__all__ = ['winDefs', 'CParser']


def winDefs(verbose=False):
    """Convenience function. Returns a parser which loads a selection of windows headers included with 
    CParser. These definitions can either be accessed directly or included before parsing
    another file like this:
        windefs = CParser.winDefs()
        p = CParser.CParser("headerFile.h", copyFrom=windefs)
    Definitions are pulled from a selection of header files included in Visual Studio
    (possibly not legal to distribute? Who knows.), some of which have been abridged
    because they take so long to parse. 
    """
    headerFiles = ['WinNt.h', 'WinDef.h', 'WinBase.h', 'BaseTsd.h', 'WTypes.h', 'WinUser.h']
    d = os.path.dirname(__file__)
    p = CParser(
        [os.path.join(d, 'headers', h) for h in headerFiles],
        types={'__int64': ('long long')},
        macros={'_WIN32': '', '_MSC_VER': '800', 'CONST': 'const', 'NO_STRICT': None},
        processAll=False
    )
    p.processAll(cache=os.path.join(d, 'headers', 'WinDefs.cache'), noCacheWarning=True, verbose=verbose)
    return p


class CParser():
    """Class for parsing C code to extract variable, struct, enum, and function declarations as well as preprocessor macros. This is not a complete C parser; instead, it is meant to simplify the process
    of extracting definitions from header files in the absence of a complete build system. Many files 
    will require some amount of manual intervention to parse properly (see 'replace' and extra arguments 
    to __init__)
    
    Usage:
        ## create parser object, load two files
        p = CParser(['header1.h', 'header2.h'])
        
        ## remove comments, preprocess, and search for declarations
        p.processAll()
        
        ## just to see what was successfully parsed from the files
        p.printAll() 
        
        ## access parsed declarations 
        allValues = p.defs['values']
        functionSignatures = p.defs['functions']
        ...
        
        ## To see what was not successfully parsed:
        unp = p.processAll(returnUnparsed=True)
        for s in unp:
            print s
    """
    
    cacheVersion = 22    ## increment every time cache structure or parsing changes to invalidate old cache files.
    
    def __init__(self, files=None, replace=None, copyFrom=None, processAll=True, cache=None, checkCache=False, verbose=False, **args):
        """Create a C parser object fiven a file or list of files. Files are read to memory and operated
        on from there.
        
            *copyFrom* may be another CParser object from which definitions should be copied.
            *replace* may be specified to perform string replacements before parsing.
               format is {'searchStr': 'replaceStr', ...}
            *cache* specifies a cache file where parsed definitions should be stored.
            *checkCache* specifies whether to attempt to reparse if it appears the header file is newer.
            Extra parameters may be used to specify the starting state of the parser. For example,
            one could provide a set of missing type declarations by
                types={'UINT': ('unsigned int'), 'STRING': ('char', 1)}
            Similarly, preprocessor macros can be specified:
                macros={'WINAPI': ''}
            
        """
        
        
        self.defs = {}  ## holds all definitions
        self.fileDefs = {}  ## holds definitions grouped by the file they came from
        
        self.initOpts = args.copy()
        self.initOpts['files'] = []
        self.initOpts['replace'] = {}
        
        self.dataList = ['types', 'variables', 'fnmacros', 'macros', 'structs', 'unions', 'enums', 'functions', 'values']
            
        self.verbose = False
            
        # placeholders for definitions that change during parsing
        #if hasPyParsing:
            #self.macroExpr = Forward()
            #self.fnMacroExpr = Forward()
            #self.definedType = Forward()
            #self.definedStruct = Forward()
            #self.definedEnum = Forward()
        
        self.fileOrder = []
        self.files = {}
        self.packList = {}  ## list describing struct packing rules as defined by #pragma pack
        if files is not None:
            if type(files) is str:
                files = [files]
            for f in files:
                self.loadFile(f, replace)
                    
        ## initialize empty definition lists
        for k in self.dataList:
            self.defs[k] = {}
            #for f in files:
                #self.fileDefs[f][k] = {}
                
        self.compiledTypes = {}  ## holds translations from typedefs/structs/unions to fundamental types
                
        self.currentFile = None
        
        # Import extra arguments if specified
        for t in args:
            for k in args[t].keys():
                self.addDef(t, k, args[t][k])
        
        # Import from other CParsers if specified
        if copyFrom is not None:
            if type(copyFrom) not in [list, tuple]:
                copyFrom = [copyFrom]
            for p in copyFrom:
                self.importDict(p.fileDefs, p.fileOrder)
                
        if processAll:
            self.processAll(cache=cache, verbose=verbose, checkCache=checkCache)
    
    def processAll(self, cache=None, returnUnparsed=False, printAfterPreprocess=False, noCacheWarning=True, verbose=False, checkCache=False):
        """Remove comments, preprocess, and parse declarations from all files. (operates in memory; does not alter the original files)
        Returns a list of the results from parseDefs.
           'cache' may specify a file where cached results are be stored or retrieved. The cache
               is automatically invalidated if any of the arguments to __init__ are changed, or if the 
               C files are newer than the cache.
           'returnUnparsed' is passed directly to parseDefs.
           'printAfterPreprocess' is for debugging; prints the result of preprocessing each file."""
        self.verbose = verbose
        if cache is not None and self.loadCache(cache, checkValidity=checkCache):
            if verbose:
                print "Loaded cached definitions; will skip parsing."
            return  ## cached values loaded successfully, nothing left to do here
        #else:
            #print "No cache.", cache
            
        
        results = []
        if noCacheWarning or verbose:
            print "Parsing C header files (no valid cache found). This could take several minutes..."
        for f in self.fileOrder:
            #fn = os.path.basename(f)
            if self.files[f] is None:
                ## This means the file could not be loaded and there was no cache.
                raise Exception('Could not find header file "%s" or a suitable cache file.' % f)
            if verbose:
                print "Removing comments from file '%s'..." % f
            self.removeComments(f)
            if verbose:
                print "Preprocessing file '%s'..." % f
            self.preprocess(f)
            if printAfterPreprocess:
                print "===== PREPROCSSED %s =======" % f
                print self.files[f]
            if verbose:
                print "Parsing definitions in file '%s'..." % f
            results.append(self.parseDefs(f, returnUnparsed))
        
        if cache is not None:
            if verbose:
                print "Writing cache file '%s'" % cache
            self.writeCache(cache)
            
        return results
        
            
    def loadCache(self, cacheFile, checkValidity=False):
        """Load a cache file. Used internally if cache is specified in processAll().
        if checkValidity=True, then run several checks before loading the cache:
           - cache file must not be older than any source files
           - cache file must not be older than this library file
           - options recorded in cache must match options used to initialize CParser"""
        
        ## make sure cache file exists 
        if type(cacheFile) is not str:
            raise Exception("cache file option must be a string.")
        if not os.path.isfile(cacheFile):
            d = os.path.dirname(__file__)  ## If file doesn't exist, search for it in this module's path
            cacheFile = os.path.join(d, "headers", cacheFile)
            if not os.path.isfile(cacheFile):
                if self.verbose:
                    print "Can't find requested cache file."
                return False
        
        ## make sure cache is newer than all input files
        canParse = hasPyParsing
        if checkValidity:
            mtime = os.stat(cacheFile).st_mtime
            for f in self.fileOrder:
                ## if file does not exist, then it does not count against the validity of the cache.
                if os.path.isfile(f):
                    if os.stat(f).st_mtime > mtime:
                        if self.verbose:
                            print "Cache file is out of date."
                        return False
                else:
                    if self.verbose:
                        print "Header file is missing: %s" % f
                    canParse = False
        
        try:
            ## read cache file
            import pickle
            cache = pickle.load(open(cacheFile, 'rb'))
            
            ## make sure __init__ options match (unless we can't parse the headers anyway)
            if checkValidity:
                if cache['opts'] != self.initOpts:
                    if self.verbose:
                        print "Cache file is not valid--created using different initialization options."
                        print cache['opts']
                        print self.initOpts
                    if canParse:
                        return False
                    elif self.verbose:
                        print "However, can't parse header files; will attempt to use the cache anyway."
                elif self.verbose:
                    print "Cache init opts are OK:"
                    print cache['opts']
                if cache['version'] < self.cacheVersion:
                    if self.verbose:
                        print "Cache file is not valid--cache format has changed."
                    if canParse:
                        return False
                    elif self.verbose:
                        print "However, can't parse header files; will attempt to use the cache anyway."

                
            ## import all parse results
            self.importDict(cache['fileDefs'], cache['fileOrder'])
            return True
        except:
            print "Warning--cache read failed:"
            sys.excepthook(*sys.exc_info())
            return False

    def importDict(self, data, order):
        """Import definitions from a dictionary. The dict format should be the
        same as CParser.fileDefs. Used internally; does not need to be called
        manually."""
        for f in order:
            f = os.path.split(f)[1]
            self.currentFile = f
            for k in self.dataList:
                for n in data[f][k]:
                    self.addDef(k, n, data[f][k][n])

    def writeCache(self, cacheFile):
        """Store all parsed declarations to cache. Used internally."""
        cache = {}
        cache['opts'] = self.initOpts
        cache['fileDefs'] = self.fileDefs
        cache['fileOrder'] = self.fileOrder
        cache['version'] = self.cacheVersion
        #for k in self.dataList:
            #cache[k] = getattr(self, k)
        import pickle
        pickle.dump(cache, open(cacheFile, 'wb'))

    def loadFile(self, file, replace=None):
        """Read a file, make replacements if requested. Called by __init__, should
        not be called manually."""
        if not os.path.isfile(file):
            ## Not a fatal error since we might be able to function properly if there is a cache file..
            #raise Exception("File %s not found" % file)
            print "Warning: C header '%s' is missing; this may cause trouble." % file
            self.files[file] = None
            return False
            
        fd = open(file, 'rU')  ## U causes all newline types to be converted to \n
        self.files[file] = fd.read()
        fd.close()
        
        if replace is not None:
            for s in replace:
                self.files[file] = re.sub(s, replace[s], self.files[file])
        self.fileOrder.append(file)
        bn = os.path.basename(file)
        self.initOpts['replace'][bn] = replace
        self.initOpts['files'].append(bn) # only interested in the file names; the directory may change between systems.
        return True
    




    #### Beginning of processing functions
    
    def assertPyparsing(self):
        """Make sure pyparsing module is available."""
        global hasPyParsing
        if not hasPyParsing:
            raise Exception("CParser class requires 'pyparsing' library for actual parsing work. Without this library, CParser can only be used with previously cached parse results.")
    

    def removeComments(self, file):
        """Remove all comments from file. (operates in memory; does not alter the original files)"""
        self.assertPyparsing()
        text = self.files[file]
        cplusplusLineComment = Literal("//") + restOfLine
        # match quoted strings first to prevent matching comments inside quotes
        self.files[file] = (quotedString | cStyleComment.suppress() | cplusplusLineComment.suppress()).transformString(text)
        
    
    def preprocess(self, file):
        """Scan named file for preprocessor directives, removing them while expanding macros. (operates in memory; does not alter the original files)"""
        self.assertPyparsing()
        self.buildParser()  ## we need this so that evalExpr works properly
        self.currentFile = file
        packStack = [(None,None)]  ## stack for #pragma pack push/pop
        self.packList[file] = [(0,None)]
        packing = None  ## current packing value 
        
        text = self.files[file]
        
        ## First join together lines split by \\n
        text = Literal('\\\n').suppress().transformString(text)
        
        #self.ppDirective = Combine("#" + Word(alphas).leaveWhitespace()) + restOfLine
        
        # define the structure of a macro definition
        name = Word(alphas+'_', alphanums+'_')('name')
        self.ppDefine = name.setWhitespaceChars(' \t')("macro") + Optional(lparen + delimitedList(name) + rparen).setWhitespaceChars(' \t')('args') + SkipTo(LineEnd())('value')
        self.ppDefine.setParseAction(self.processMacroDefn)
        
        #self.updateMacroDefns()
        #self.updateFnMacroDefns()

        # define pattern for scanning through the input string
        #self.macroExpander = (self.macroExpr | self.fnMacroExpr)
        
        ## Comb through lines, process all directives
        lines = text.split('\n')
        
        result = []
        #macroExpander = (quotedString | self.macroExpander)
        directive = re.compile(r'\s*#([a-zA-Z]+)(.*)$')
        ifTrue = [True]
        ifHit = []
        for i in range(len(lines)):
            line = lines[i]
            newLine = ''
            m = directive.match(line)
            if m is None:  # regular code line
                if ifTrue[-1]:  # only include if we are inside the correct section of an IF block
                    #line = macroExpander.transformString(line)  # expand all known macros
                    newLine = self.expandMacros(line)
            else:  # macro line
                d = m.groups()[0]
                rest = m.groups()[1]
                
                #print "PREPROCESS:", d, rest
                if d == 'ifdef':
                    d = 'if'
                    rest = 'defined '+rest
                elif d == 'ifndef':
                    d = 'if'
                    rest = '!defined '+rest
                    
                ## Evaluate 'defined' operator before expanding macros
                if d in ['if', 'elif']:
                    def pa(t):
                        return ['0', '1'][t['name'] in self.defs['macros'] or t['name'] in self.defs['fnmacros']]
                    rest = (
                        Keyword('defined') + 
                        (name | lparen + name + rparen)
                    ).setParseAction(pa).transformString(rest)
                elif d in ['define', 'undef']:    
                    macroName, rest = re.match(r'\s*([a-zA-Z_][a-zA-Z0-9_]*)(.*)$', rest).groups()
                
                ## Expand macros if needed
                if rest is not None and (all(ifTrue) or d in ['if', 'elif']):
                    rest = self.expandMacros(rest)
                    
                if d == 'elif':
                    if ifHit[-1] or not all(ifTrue[:-1]):
                        ev = False
                    else:
                        ev = self.evalPreprocessorExpr(rest)
                    if self.verbose:
                        print "  "*(len(ifTrue)-2) + line, rest, ev
                    ifTrue[-1] = ev
                    ifHit[-1] = ifHit[-1] or ev
                elif d == 'else':
                    if self.verbose:
                        print "  "*(len(ifTrue)-2) + line, not ifHit[-1]
                    ifTrue[-1] = (not ifHit[-1]) and all(ifTrue[:-1])
                    ifHit[-1] = True
                elif d == 'endif':
                    ifTrue.pop()
                    ifHit.pop()
                    if self.verbose:
                        print "  "*(len(ifTrue)-1) + line
                elif d == 'if':
                    if all(ifTrue):
                        ev = self.evalPreprocessorExpr(rest)
                    else:
                        ev = False
                    if self.verbose:
                        print "  "*(len(ifTrue)-1) + line, rest, ev
                    ifTrue.append(ev)
                    ifHit.append(ev)
                elif d == 'define':
                    if not ifTrue[-1]:
                        continue
                    if self.verbose:
                        print "  "*(len(ifTrue)) + "define:", macroName, rest
                    try:
                        self.ppDefine.parseString(macroName+ ' ' + rest) ## macro is registered here
                    except:
                        print "Error processing macro definition:", macroName, rest
                        print "      ", sys.exc_info()[1]
                elif d == 'undef':
                    if not ifTrue[-1]:
                        continue
                    try:
                        self.remDef('macros', macroName.strip())
                        #self.macroListString = '|'.join(self.defs['macros'].keys() + self.defs['fnmacros'].keys())
                        #self.updateMacroDefns()
                    except:
                        if sys.exc_info()[0] is not KeyError:
                            sys.excepthook(*sys.exc_info())
                            print "Error removing macro definition '%s'" % macroName.strip()
                elif d == 'pragma':  ## Check for changes in structure packing
                    if not ifTrue[-1]:
                        continue
                    m = re.match(r'\s+pack\s*\(([^\)]+)\)', rest)
                    if m is None:
                        continue
                    opts = [s.strip() for s in m.groups()[0].split(',')]
                    
                    pushpop = id = val = None
                    for o in opts:
                        if o in ['push', 'pop']:
                            pushpop = o
                        elif o.isdigit():
                            val = int(o)
                        else:
                            id = o
                            
                    if val is not None:
                        packing = val
                        
                    if pushpop == 'push':
                        packStack.append((packing, id))
                    elif opts[0] == 'pop':
                        if id is None:
                            packStack.pop()
                        else:
                            ind = None
                            for i in range(len(packStack)):
                                if packStack[i][1] == id:
                                    ind = i
                                    break
                            if ind is not None:
                                packStack = packStack[:ind]
                        if val is None:
                            packing = packStack[-1][0]
                    else:
                        packing = int(opts[0])
                    
                    if self.verbose:
                        print ">> Packing changed to %s at line %d" % (str(packing), i)
                    self.packList[file].append((i, packing))
                else:
                    pass  ## Ignore any other directives
                    
            result.append(newLine)      
        self.files[file] = '\n'.join(result)
        
    def evalPreprocessorExpr(self, expr):
        ## make a few alterations so the expression can be eval'd
        macroDiffs = (
            Literal('!').setParseAction(lambda: ' not ') | 
            Literal('&&').setParseAction(lambda: ' and ') | 
            Literal('||').setParseAction(lambda: ' or ') | 
            Word(alphas+'_',alphanums+'_').setParseAction(lambda: '0'))
        expr2 = macroDiffs.transformString(expr)
            
        try:
            ev = bool(eval(expr2))
        except:
            if self.verbose:
                print "Error evaluating preprocessor expression: %s [%s]" % (expr, expr2)
                print "      ", sys.exc_info()[1]
            ev = False
        return ev
            
        
        
    #def updateMacroDefns(self):
        ##self.macroExpr << MatchFirst( [Keyword(m)('macro') for m in self.defs['macros']] )
        ##self.macroExpr.setParseAction(self.processMacroRef)
        
        ## regex is faster than pyparsing.
        ## Matches quoted strings and macros
        
        ##names = self.defs['macros'].keys() + self.defs['fnmacros'].keys()
        #if len(self.macroListString) == 0:
            #self.macroRegex = None
        #else:
            #self.macroRegex = re.compile(
                #r'("(\\"|[^"])*")|(\b(%s)\b)'   %   self.macroListString
            #)

    #def updateFnMacroDefns(self):
        #self.fnMacroExpr << MatchFirst( [(Keyword(m)('macro') + lparen + Group(delimitedList(expression))('args') + rparen) for m in self.defs['fnmacros']] )
        #self.fnMacroExpr.setParseAction(self.processFnMacroRef)        
        
    
    def processMacroDefn(self, t):
        """Parse a #define macro and register the definition"""
        if self.verbose:
            print "MACRO:", t
        #macroVal = self.macroExpander.transformString(t.value).strip()
        #macroVal = Literal('\\\n').suppress().transformString(macroVal) ## remove escaped newlines
        macroVal = t.value.strip()
        if macroVal in self.defs['fnmacros']:
            self.addDef('fnmacros', t.macro, self.defs['fnmacros'][macroVal])
            if self.verbose:
                print "  Copy fn macro %s => %s" % (macroVal, t.macro) 
        else:
            if t.args == '':
                val = self.evalExpr(macroVal)
                self.addDef('macros', t.macro, macroVal)
                self.addDef('values', t.macro, val)
                if self.verbose:
                    print "  Add macro:", t.macro, "("+str(val)+")", self.defs['macros'][t.macro]
            else:
                self.addDef('fnmacros', t.macro,  self.compileFnMacro(macroVal, [x for x in t.args]))
                if self.verbose:
                    print "  Add fn macro:", t.macro, t.args, self.defs['fnmacros'][t.macro]
        
        #if self.macroListString == '':
            #self.macroListString = t.macro
        #else:
            #self.macroListString += '|' + t.macro
        #self.updateMacroDefns()
        #self.macroExpr << MatchFirst( map(Keyword,self.defs['macros'].keys()) )
        return "#define " + t.macro + " " + macroVal
    
    
    def compileFnMacro(self, text, args):
        """Turn a function macro spec into a compiled description"""
        ## find all instances of each arg in text
        argRegex = re.compile(r'("(\\"|[^"])*")|(\b(%s)\b)'   %   ('|'.join(args)))
        start = 0
        parts = []
        argOrder = []
        N = 3
        for m in argRegex.finditer(text):
            arg = m.groups()[N]
            #print m, arg
            if arg is not None:
                parts.append(text[start:m.start(N)] + '%s')
                start = m.end(N)
                argOrder.append(args.index(arg))
        parts.append(text[start:])
        return (''.join(parts), argOrder)
        
    
    def expandMacros(self, line):
        reg = re.compile(r'("(\\"|[^"])*")|(\b(\w+)\b)')
        parts = []
        start = 0
        N = 3   ## the group number to check for macro names
        macros = self.defs['macros']
        fnmacros = self.defs['fnmacros']
        for m in reg.finditer(line):
            name = m.groups()[N]
            if name in macros:
                parts.append(line[start:m.start(N)])
                start = m.end(N)
                parts.append(macros[name])
            elif name in fnmacros:
                try:  ## If function macro expansion fails, just ignore it.
                    exp, end = self.expandFnMacro(name, line[m.end(N):])
                    parts.append(line[start:m.start(N)])
                    start = end + m.end(N)
                    parts.append(exp)
                except:
                    if sys.exc_info()[1][0] != 0:
                        print "Function macro expansion failed:", name, line[m.end(N):]
                        raise
        parts.append(line[start:])
        return ''.join(parts)
                


    #def expandMacros(self, line):
        #if self.macroRegex is None:
            #return line
        #parts = []
        #start = 0
        #N = 3   ## the group number to check for macro names
        #for m in self.macroRegex.finditer(line):
            #name = m.groups()[N]
            #if name is not None:
                #if name in self.defs['macros']:
                    #parts.append(line[start:m.start(N)])
                    #start = m.end(N)
                    #parts.append(self.defs['macros'][name])
                #elif name in self.defs['fnmacros']:
                    #try:  ## If function macro expansion fails, just ignore it.
                        #exp, end = self.expandFnMacro(name, line[m.end(N):])
                        #parts.append(line[start:m.start(N)])
                        #start = end + m.end(N)
                        #parts.append(exp)
                    #except:
                        #if sys.exc_info()[1][0] != 0:
                            #print "Function macro expansion failed:", name, line[m.end(N):]
                            #raise
                        
                #else:
                    #raise Exception("Macro '%s' not found (internal error)" % name)
        #parts.append(line[start:])
        #return ''.join(parts)

    def expandFnMacro(self, name, text):
        #print "expandMacro:", name, text
        defn = self.defs['fnmacros'][name]
        ## defn looks like ('%s + %s / %s', (0, 0, 1))
        
        argList = stringStart + lparen + Group(delimitedList(expression))('args') + rparen
        res = [x for x in argList.scanString(text, 1)]
        if len(res) == 0:
            raise Exception(0, "Function macro '%s' not followed by (...)" % name)
        args, start, end = res[0]
        #print "  ", res 
        #print "  ", args
        #print "  ", defn
        newStr = defn[0] % tuple([args[0][i] for i in defn[1]])
        #print "  ", newStr
        return (newStr, end)
        
        
    # parse action to replace macro references with their respective definition
    #def processMacroRef(self, t):
        #return self.defs['macros'][t.macro]
            
    #def processFnMacroRef(self, t):
        #m = self.defs['fnmacros'][t.macro]
        ##print "=====>>"
        ##print "Process FN MACRO:", t
        ##print "  macro defn:", t.macro, m
        ##print "  macro call:", t.args
        ### m looks like ('a + b', ('a', 'b'))
        #newStr = m[0][:]
        ##print "  starting str:", newStr
        #try:
            #for i in range(len(m[1])):
                ##print "  step", i
                #arg = m[1][i]
                ##print "    arg:", arg, '=>', t.args[i]
                
                #newStr = Keyword(arg).copy().setParseAction(lambda: t.args[i]).transformString(newStr)
                ##print "    new str:", newStr
        #except:
            ##sys.excepthook(*sys.exc_info())
            #raise
        ##print "<<====="
        #return newStr
        
        
        
        
        
        
        
        
    def parseDefs(self, file, returnUnparsed=False):
        """Scan through the named file for variable, struct, enum, and function declarations.
        Returns the entire tree of successfully parsed tokens.
        If returnUnparsed is True, return a string of all lines that failed to match (for debugging)."""
        self.assertPyparsing()
        self.currentFile = file
        #self.definedType << kwl(self.defs['types'].keys())
    
        parser = self.buildParser()
        if returnUnparsed:
            text = parser.suppress().transformString(self.files[file])
            return re.sub(r'\n\s*\n', '\n', text)
        else:
            return [x[0] for x in parser.scanString(self.files[file])]

    def buildParser(self):
        """Builds the entire tree of parser elements for the C language (the bits we support, anyway).
        """
        
        if hasattr(self, 'parser'):
            return self.parser
            
            
        self.assertPyparsing()
        
        
        self.structType = Forward()
        self.enumType = Forward()
        self.typeSpec = (typeQualifier + (
            fundType | 
            Optional(kwl(sizeModifiers + signModifiers)) + ident | 
            self.structType | 
            self.enumType
        ) + typeQualifier + msModifier).setParseAction(recombine)
        #self.argList = Forward()
        
        ### Abstract declarators for use in function pointer arguments
        #   Thus begins the extremely hairy business of parsing C declarators. 
        #   Whomever decided this was a reasonable syntax should probably never breed.
        #   The following parsers combined with the processDeclarator function
        #   allow us to turn a nest of type modifiers into a correctly
        #   ordered list of modifiers.
        
        self.declarator = Forward()
        self.abstractDeclarator = Forward()
        
        ## abstract declarators look like:
        #     <empty string>
        #     *
        #     **[num]
        #     (*)(int, int)
        #     *( )(int, int)[10]
        #     ...etc...
        self.abstractDeclarator << Group(
            typeQualifier + Group(ZeroOrMore('*'))('ptrs') + typeQualifier +
            ((Optional('&')('ref')) | (lparen + self.abstractDeclarator + rparen)('center')) + 
            Optional(lparen + Optional(delimitedList(Group(
                self.typeSpec('type') + 
                self.abstractDeclarator('decl') + 
                Optional(Literal('=').suppress() + expression, default=None)('val')
            )), default=None) + rparen)('args') + 
            Group(ZeroOrMore(lbrack + Optional(expression, default='-1') + rbrack))('arrays')
        )
        
        ## Argument list may consist of declarators or abstract declarators
        #self.argList << delimitedList(Group(
            #self.typeSpec('type') + 
            #(self.declarator('decl') | self.abstractDeclarator('decl')) + 
            #Optional(Keyword('=')) + expression
        #))

        ## declarators look like:
        #     varName
        #     *varName
        #     **varName[num]
        #     (*fnName)(int, int)
        #     * fnName(int arg1=0)[10]
        #     ...etc...
        self.declarator << Group(
            typeQualifier + callConv + Group(ZeroOrMore('*'))('ptrs') + typeQualifier +
            ((Optional('&')('ref') + ident('name')) | (lparen + self.declarator + rparen)('center')) + 
            Optional(lparen + Optional(delimitedList(Group(
                self.typeSpec('type') + 
                (self.declarator | self.abstractDeclarator)('decl') + 
                Optional(Literal('=').suppress() + expression, default=None)('val')
            )), default=None) + rparen)('args') + 
            Group(ZeroOrMore(lbrack + Optional(expression, default='-1') + rbrack))('arrays')
        )
        self.declaratorList = Group(delimitedList(self.declarator))

        ## typedef
        self.typeDecl = Keyword('typedef') + self.typeSpec('type') + self.declaratorList('declList') + semi
        self.typeDecl.setParseAction(self.processTypedef)

        ## variable declaration
        self.variableDecl = Group(self.typeSpec('type') + Optional(self.declaratorList('declList')) + Optional(Literal('=').suppress() + (expression('value') | (lbrace + Group(delimitedList(expression))('arrayValues') + rbrace)))) + semi
        
        self.variableDecl.setParseAction(self.processVariable)
        
        ## function definition
        #self.paramDecl = Group(self.typeSpec + (self.declarator | self.abstractDeclarator)) + Optional(Literal('=').suppress() + expression('value'))
        self.typelessFunctionDecl = self.declarator('decl') + nestedExpr('{', '}').suppress()
        self.functionDecl = self.typeSpec('type') + self.declarator('decl') + nestedExpr('{', '}').suppress()
        self.functionDecl.setParseAction(self.processFunction)
        
        
        ## Struct definition
        self.structDecl = Forward()
        structKW = (Keyword('struct') | Keyword('union'))
        #self.structType << structKW('structType') + ((Optional(ident)('name') + lbrace + Group(ZeroOrMore( Group(self.structDecl | self.variableDecl.copy().setParseAction(lambda: None)) ))('members') + rbrace) | ident('name'))
        self.structMember = (
            Group(self.variableDecl.copy().setParseAction(lambda: None)) |
            (self.typeSpec + self.declarator + nestedExpr('{', '}')).suppress() |
            (self.declarator + nestedExpr('{', '}')).suppress()
        )
        self.declList = lbrace + Group(OneOrMore(self.structMember))('members') + rbrace
        self.structType << (Keyword('struct') | Keyword('union'))('structType') + ((Optional(ident)('name') + self.declList) | ident('name'))
        
        self.structType.setParseAction(self.processStruct)
        #self.updateStructDefn()
        
        self.structDecl = self.structType + semi

        ## enum definition
        enumVarDecl = Group(ident('name')  + Optional(Literal('=').suppress() + (integer('value') | ident('valueName'))))
        
        self.enumType << Keyword('enum') + (Optional(ident)('name') + lbrace + Group(delimitedList(enumVarDecl))('members') + rbrace | ident('name'))
        self.enumType.setParseAction(self.processEnum)
        
        self.enumDecl = self.enumType + semi

        
        #self.parser = (self.typeDecl | self.variableDecl | self.structDecl | self.enumDecl | self.functionDecl)
        self.parser = (self.typeDecl | self.variableDecl | self.functionDecl)
        return self.parser
    
    def processDeclarator(self, decl):
        """Process a declarator (without base type) and return a tuple (name, [modifiers])
        See processType(...) for more information."""
        toks = []
        name = None
        #print "DECL:", decl
        if 'callConv' in decl and len(decl['callConv']) > 0:
            toks.append(decl['callConv'])
        if 'ptrs' in decl and len(decl['ptrs']) > 0:
            toks.append('*' * len(decl['ptrs']))
        if 'arrays' in decl and len(decl['arrays']) > 0:
            #arrays  = []
            #for x in decl['arrays']:
                #n = self.evalExpr(x)
                #if n == -1:           ## If an array was given as '[]', interpret it as '*' instead.
                    #toks.append('*')
                #else:
                    #arrays.append(n)
            #if len(arrays) > 0:
                #toks.append(arrays)
            toks.append([self.evalExpr(x) for x in decl['arrays']])
        if 'args' in decl and len(decl['args']) > 0:
            #print "  process args"
            if decl['args'][0] is None:
                toks.append(())
            else:
                toks.append(tuple([self.processType(a['type'], a['decl']) + (a['val'][0],) for a in decl['args']]))
        if 'ref' in decl:
            toks.append('&')
        if 'center' in decl:
            (n, t) = self.processDeclarator(decl['center'][0])
            if n is not None:
                name = n
            toks.extend(t)
        if 'name' in decl:
            name = decl['name']
        return (name, toks)
    
    def processType(self, typ, decl):
        """Take a declarator + base type and return a serialized name/type description.
        The description will be a list of elements (name, [basetype, modifier, modifier, ...])
          - name is the string name of the declarator or None for an abstract declarator
          - basetype is the string representing the base type
          - modifiers can be:
             '*'    - pointer (multiple pointers "***" allowed)
             '&'    - reference
             '__X'  - calling convention (windows only). X can be 'cdecl' or 'stdcall' 
             list   - array. Value(s) indicate the length of each array, -1 for incomplete type.
             tuple  - function, items are the output of processType for each function argument.
             
        Examples:
            int *x[10]            =>  ('x', ['int', [10], '*'])
            char fn(int x)         =>  ('fn', ['char', [('x', ['int'])]])
            struct s (*)(int, int*)   =>  (None, ["struct s", ((None, ['int']), (None, ['int', '*'])), '*'])
        """
        #print "PROCESS TYPE/DECL:", typ, decl
        (name, decl) = self.processDeclarator(decl)
        return (name, [typ] + decl)
        
            

    def processEnum(self, s, l, t):
        try:
            if self.verbose:
                print "ENUM:", t
            if t.name == '':
                n = 0
                while True:
                    name = 'anonEnum%d' % n
                    if name not in self.defs['enums']:
                        break
                    n += 1
            else:
                name = t.name[0]
                
            if self.verbose:
                print "  name:", name
                
            if name not in self.defs['enums']:
                i = 0
                enum = {}
                for v in t.members:
                    if v.value != '':
                        i = eval(v.value)
                    if v.valueName != '':
                        i = enum[v.valueName]
                    enum[v.name] = i
                    self.addDef('values', v.name, i)
                    i += 1
                if self.verbose:
                        print "  members:", enum
                self.addDef('enums', name, enum)
                self.addDef('types', 'enum '+name, ('enum', name))
            return ('enum ' + name)
        except:
            if self.verbose:
                print "Error processing enum:", t
            sys.excepthook(*sys.exc_info())


    def processFunction(self, s, l, t):
        if self.verbose:
            print "FUNCTION", t, t.keys()
        
        try:
            (name, decl) = self.processType(t.type, t.decl[0])
            if len(decl) == 0 or type(decl[-1]) != tuple:
                print t
                raise Exception("Incorrect declarator type for function definition.")
            if self.verbose:
                print "  name:", name
                print "  sig:", decl
            self.addDef('functions', name, (decl[:-1], decl[-1]))
            
        except:
            if self.verbose:
                print "Error processing function:", t
            sys.excepthook(*sys.exc_info())


    def packingAt(self, line):
        """Return the structure packing value at the given line number"""
        packing = None
        for p in self.packList[self.currentFile]:
            if p[0] <= line: 
                packing = p[1]
            else:
                break
        return packing

    def processStruct(self, s, l, t):
        try:
            strTyp = t.structType  # struct or union
            
            ## check for extra packing rules
            packing = self.packingAt(lineno(l, s))
            
            if self.verbose:
                print strTyp.upper(), t.name, t
            if t.name == '':
                n = 0
                while True:
                    sname = 'anon_%s%d' % (strTyp, n)
                    if sname not in self.defs[strTyp+'s']:
                        break
                    n += 1
            else:
                if type(t.name) is str:
                    sname = t.name
                else:
                    sname = t.name[0]
            if self.verbose:
                print "  NAME:", sname
            if sname not in self.defs[strTyp+'s'] or self.defs[strTyp+'s'][sname].get('members', []) == []:
                if self.verbose:
                    print "  NEW " + strTyp.upper()
                struct = []
                for m in t.members:
                    typ = m[0].type
                    val = self.evalExpr(m)
                    if self.verbose:
                        print "    member:", m, m[0].keys(), m[0].declList
                    if len(m[0].declList) == 0:  ## anonymous member
                        struct.append((None, [typ], None))
                    for d in m[0].declList:
                        (name, decl) = self.processType(typ, d)
                        struct.append((name, decl, val))
                        if self.verbose:
                            print "      ", name, decl, val
                self.addDef(strTyp+'s', sname, {'pack': packing, 'members': struct})
                self.addDef('types', strTyp+' '+sname, (strTyp, sname))
                #self.updateStructDefn()
            else:
                if self.verbose:
                    print "  (already defined)"
            return strTyp+' '+sname
        except:
            #print t
            sys.excepthook(*sys.exc_info())

    def processVariable(self, s, l, t):
        if self.verbose:
            print "VARIABLE:", t
        try:
            val = self.evalExpr(t[0])
            for d in t[0].declList:
                (name, typ) = self.processType(t[0].type, d)
                if type(typ[-1]) is tuple:  ## this is a function prototype
                    if self.verbose:
                        print "  Add function prototype:", name, typ, val
                    self.addDef('functions', name, (typ[:-1], typ[-1]))
                else:
                    if self.verbose:
                        print "  Add variable:", name, typ, val
                    self.addDef('variables', name, (val, typ))
                    self.addDef('values', name, val)
        except:
            #print t, t[0].name, t.value
            sys.excepthook(*sys.exc_info())

    def processTypedef(self, s, l, t):
        if self.verbose:
            print "TYPE:", t
        typ = t.type
        #print t, t.type
        for d in t.declList:
            (name, decl) = self.processType(typ, d)
            if self.verbose:
                print "  ", name, decl
            self.addDef('types', name, decl)
            #self.definedType << MatchFirst( map(Keyword,self.defs['types'].keys()) )
        
    def evalExpr(self, toks):
        ## Evaluates expressions. Currently only works for expressions that also 
        ## happen to be valid python expressions.
        ## This function does not currently include previous variable
        ## declarations, but that should not be too difficult to implement..
        #print "Eval:", toks
        try:
            if isinstance(toks, basestring):
                #print "  as string"
                val = self.eval(toks, None, self.defs['values'])
            elif toks.arrayValues != '':
                #print "  as list:", toks.arrayValues
                val = [self.eval(x, None, self.defs['values']) for x in toks.arrayValues]
            elif toks.value != '':
                #print "  as value"
                val = self.eval(toks.value, None, self.defs['values'])
            else:
                #print "  as None"
                val = None
            return val
        except:
            if self.verbose:
                print "    failed eval:", toks
                print "                ", sys.exc_info()[1]
            return None
            
    def eval(self, expr, *args):
        """Just eval with a little extra robustness."""
        expr = expr.strip()
        cast = (lparen + self.typeSpec + self.abstractDeclarator + rparen).suppress()
        expr = (quotedString | number | cast).transformString(expr)
        if expr == '':
            return None
        return eval(expr, *args)
        
    def printAll(self, file=None):
        """Print everything parsed from files. Useful for debugging."""
        from pprint import pprint
        for k in self.dataList:
            print "============== %s ==================" % k
            if file is None:
                pprint(self.defs[k])
            else:
                pprint(self.fileDefs[file][k])
                
    def addDef(self, typ, name, val):
        """Add a definition of a specific type to both the definition set for the current file and the global definition set."""
        self.defs[typ][name] = val
        if self.currentFile is None:
            baseName = None
        else:
            baseName = os.path.basename(self.currentFile)
        if baseName not in self.fileDefs:
            self.fileDefs[baseName] = {}
            for k in self.dataList:
                self.fileDefs[baseName][k] = {}
        self.fileDefs[baseName][typ][name] = val

    def remDef(self, typ, name):
        if self.currentFile is None:
            baseName = None
        else:
            baseName = os.path.basename(self.currentFile)
        del self.defs[typ][name]
        del self.fileDefs[baseName][typ][name]
        

    def isFundType(self, typ):
        """Return True if this type is a fundamental C type, struct, or union"""
        if typ[0][:7] == 'struct ' or typ[0][:6] == 'union ' or typ[0][:5] == 'enum ':
            return True
            
        names = baseTypes + sizeModifiers + signModifiers
        for w in typ[0].split():
            if w not in names:
                return False
        return True

    def evalType(self, typ):
        """evaluate a named type into its fundamental type"""
        used = []
        while True:
            if self.isFundType(typ):
                ## remove 'signed' before returning evaluated type
                typ[0] = re.sub(r'\bsigned\b', '', typ[0]).strip()
                
                
                return typ
            parent = typ[0]
            if parent in used:
                raise Exception('Recursive loop while evaluating types. (typedefs are %s)' % (' -> '.join(used+[parent])))
            used.append(parent)
            if not parent in self.defs['types']:
                raise Exception('Unknown type "%s" (typedefs are %s)' % (parent, ' -> '.join(used)))
            pt = self.defs['types'][parent]
            typ = pt + typ[1:]

    def find(self, name):
        """Search all definitions for the given name"""
        res = []
        for f in self.fileDefs:
            fd = self.fileDefs[f]
            for t in fd:
                typ = fd[t]
                for k in typ:
                    if isinstance(name, basestring):
                        if k == name:
                            res.append((f, t))
                    else:
                        if re.match(name, k):
                            res.append((f, t, k))
        return res

    
    
    def findText(self, text):
        """Search all file strings for text, return matching lines."""
        res = []
        for f in self.files:
            l = self.files[f].split('\n')
            for i in range(len(l)):
                if text in l[i]:
                    res.append((f, i, l[i]))
        return res
        
        
hasPyParsing = False
try: 
    from pyparsing import *
    ParserElement.enablePackrat()
    hasPyParsing = True
except:
    pass  ## no need to do anything yet as we might not be using any parsing functions..


## Define some common language elements if pyparsing is available.
if hasPyParsing:
    ## Some basic definitions
    expression = Forward()
    pexpr = '(' + expression + ')'
    numTypes = ['int', 'float', 'double', '__int64']
    baseTypes = ['char', 'bool', 'void'] + numTypes
    sizeModifiers = ['short', 'long']
    signModifiers = ['signed', 'unsigned']
    qualifiers = ['const', 'static', 'volatile', 'inline', 'restrict', 'near', 'far']
    msModifiers = ['__based', '__declspec', '__fastcall', '__restrict', '__sptr', '__uptr', '__w64', '__unaligned', '__nullterminated']
    keywords = ['struct', 'enum', 'union', '__stdcall', '__cdecl'] + qualifiers + baseTypes + sizeModifiers + signModifiers

    def kwl(strs):
        """Generate a match-first list of keywords given a list of strings."""
        #return MatchFirst(map(Keyword,strs))
        return Regex(r'\b(%s)\b' % '|'.join(strs))

    keyword = kwl(keywords)
    wordchars = alphanums+'_$'
    ident = (WordStart(wordchars) + ~keyword + Word(alphas+"_",alphanums+"_$") + WordEnd(wordchars)).setParseAction(lambda t: t[0])
    #integer = Combine(Optional("-") + (Word( nums ) | Combine("0x" + Word(hexnums)))) 
    semi   = Literal(";").ignore(quotedString).suppress()
    lbrace = Literal("{").ignore(quotedString).suppress()
    rbrace = Literal("}").ignore(quotedString).suppress()
    lbrack = Literal("[").ignore(quotedString).suppress()
    rbrack = Literal("]").ignore(quotedString).suppress()
    lparen = Literal("(").ignore(quotedString).suppress()
    rparen = Literal(")").ignore(quotedString).suppress()
    hexint = Regex('-?0x[%s]+[UL]*'%hexnums).setParseAction(lambda t: t[0].rstrip('UL'))
    decint = Regex(r'-?\d+[UL]*').setParseAction(lambda t: t[0].rstrip('UL'))
    integer = (hexint | decint)
    floating = Regex(r'-?((\d+(\.\d*)?)|(\.\d+))([eE]-?\d+)?')
    number = (hexint | floating | decint)
    bitfieldspec = ":" + integer
    biOperator = oneOf("+ - / * | & || && ! ~ ^ % == != > < >= <= -> . :: << >> = ? :")
    uniRightOperator = oneOf("++ --")
    uniLeftOperator = oneOf("++ -- - + * sizeof new")
    name = (WordStart(wordchars) + Word(alphas+"_",alphanums+"_$") + WordEnd(wordchars))
    #number = Word(hexnums + ".-+xUL").setParseAction(lambda t: t[0].rstrip('UL'))
    #stars = Optional(Word('*&'), default='')('ptrs')  ## may need to separate & from * later?
    callConv = Optional(Keyword('__cdecl')|Keyword('__stdcall'))('callConv')
    
    ## Removes '__name' from all type specs.. may cause trouble.
    underscore2Ident = (WordStart(wordchars) + ~keyword + '__' + Word(alphanums,alphanums+"_$") + WordEnd(wordchars)).setParseAction(lambda t: t[0])
    typeQualifier = ZeroOrMore((underscore2Ident + Optional(nestedExpr())) | kwl(qualifiers)).suppress()
    
    msModifier = ZeroOrMore(kwl(msModifiers) + Optional(nestedExpr())).suppress()
    pointerOperator = (
        '*' + typeQualifier |
        '&' + typeQualifier |
        '::' + ident + typeQualifier
    )


    ## language elements
    fundType = OneOrMore(kwl(signModifiers + sizeModifiers + baseTypes)).setParseAction(lambda t: ' '.join(t))



    ## Is there a better way to process expressions with cast operators??
    castAtom = (
        ZeroOrMore(uniLeftOperator) + Optional('('+ident+')').suppress() + 
        ((
            ident + '(' + Optional(delimitedList(expression)) + ')' | 
            ident + OneOrMore('[' + expression + ']') | 
            ident | number | quotedString
        )  |
        ('(' + expression + ')')) + 
        ZeroOrMore(uniRightOperator)
    )
    uncastAtom = (
        ZeroOrMore(uniLeftOperator) + 
        ((
            ident + '(' + Optional(delimitedList(expression)) + ')' | 
            ident + OneOrMore('[' + expression + ']') | 
            ident | number | quotedString
        )  |
        ('(' + expression + ')')) + 
        ZeroOrMore(uniRightOperator)
    )
    atom = castAtom | uncastAtom

    expression << Group(
        atom + ZeroOrMore(biOperator + atom)
    )
    arrayOp = lbrack + expression + rbrack

    def recombine(tok):
        """Flattens a tree of tokens and joins into one big string."""
        return " ".join(flatten(tok.asList()))
    expression.setParseAction(recombine)
            
    def flatten(lst):
        res = []
        for i in lst:
            if type(i) in [list, tuple]:
                res.extend(flatten(i))
            else:
                res.append(str(i))
        return res

    def printParseResults(pr, depth=0, name=''):
        """For debugging; pretty-prints parse result objects."""
        start = name + " "*(20-len(name)) + ':'+ '..'*depth    
        if isinstance(pr, ParseResults):
            print start
            for i in pr:
                name = ''
                for k in pr.keys():
                    if pr[k] is i:
                        name = k
                        break
                printParseResults(i, depth+1, name)
        else:
            print start  + str(pr)



## Just for fun..
if __name__ == '__main__':
    files = sys.argv[1:]
    p = CParser(files)
    p.processAll()
    p.printAll()
    