[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[GNUnet-SVN] r25898 - monkey/trunk/seaspider
|
From: |
gnunet |
|
Subject: |
[GNUnet-SVN] r25898 - monkey/trunk/seaspider |
|
Date: |
Fri, 25 Jan 2013 14:31:10 +0100 |
Author: teichm
Date: 2013-01-25 14:31:10 +0100 (Fri, 25 Jan 2013)
New Revision: 25898
Added:
monkey/trunk/seaspider/ignorefile.example
Modified:
monkey/trunk/seaspider/seasp2_convert
Log:
ignorefile format and parsing added
Added: monkey/trunk/seaspider/ignorefile.example
===================================================================
--- monkey/trunk/seaspider/ignorefile.example (rev 0)
+++ monkey/trunk/seaspider/ignorefile.example 2013-01-25 13:31:10 UTC (rev
25898)
@@ -0,0 +1,14 @@
+# lines starting with a # are comments
+# other lines have to be in the format:
+# ^(filename)?:(functionname)?:pattern$
+# if filename or functionname is empty it matches any file / function.
+# leading and trailing whitespace will be ignored.
+
+# ignore every expression containing 'user_privatekey'
+::user_privatekey
+
+# ignore the local variable 'salt' in function 'encrypt_msg'
+:encrypt_msg:salt
+
+# ignore the variable 'mailto' in file libmail.c
+libmail.c::mailto
Modified: monkey/trunk/seaspider/seasp2_convert
===================================================================
--- monkey/trunk/seaspider/seasp2_convert 2013-01-25 13:05:23 UTC (rev
25897)
+++ monkey/trunk/seaspider/seasp2_convert 2013-01-25 13:31:10 UTC (rev
25898)
@@ -7,46 +7,85 @@
from collections import deque
+def parseIgnores(ignfile):
+ if not ignfile:
+ return []
+
+ ignlist = []
+ lineno = 0
+
+ lines = ignfile.readlines(100)
+ while lines:
+ for line in lines:
+ lineno += 1
+ line = line.strip()
+ if line == '' or line.startswith('#'):
+ continue
+ m = re.match(r'^([^:]*):([^:]*):(.+)$', line)
+ if not m or m.lastindex < 3:
+ print 'unknown ignore pattern on line %s: %s' % (lineno, line)
+ continue
+ ign_file = m.group(1)
+ ign_func = m.group(2)
+ ign_pattern = m.group(3)
+ ignlist.append((ign_file, ign_func, ign_pattern))
+ lines = ignfile.readlines(100)
+ ignfile.close()
+ return ignlist
+
# argparse Action for a readable dir
class readable_dir(argparse.Action):
- def __call__(self, parser, namespace, values, option_string=None):
- prospective_dir=values
- if not os.path.exists(prospective_dir):
- raise argparse.ArgumentTypeError("readable_dir:{0} is
not a valid path".format(prospective_dir))
- if os.access(prospective_dir, os.R_OK):
- setattr(namespace, self.dest, prospective_dir)
- else:
- raise argparse.ArgumentTypeError("readable_dir:{0} is
not a readable dir".format(prospective_dir))
+ def __call__(self, parser, namespace, values, option_string=None):
+ prospective_dir=values
+ if not os.path.exists(prospective_dir):
+ raise argparse.ArgumentTypeError("readable_dir:{0} is not a valid
path".format(prospective_dir))
+ if os.access(prospective_dir, os.R_OK):
+ setattr(namespace, self.dest, prospective_dir)
+ else:
+ raise argparse.ArgumentTypeError("readable_dir:{0} is not a
readable dir".format(prospective_dir))
# initialize argument parser
parser = argparse.ArgumentParser(description="converts .sea files generated by
cparser to a seaspider sqlite database")
parser.add_argument('--version', action='version', version='db_convert 1.0')
-parser.add_argument("-o", "--output", metavar="FILENAME",
type=argparse.FileType('w'), dest="outfile", help="filename for the resulting
sqlite database. defaults to INPUT.sqlite")
-parser.add_argument('INPUT', nargs='?', action=readable_dir,
default=os.getcwd(), help='input file or directory which is traversed
recursively. defaults to PWD')
+parser.add_argument("-o", "--output", metavar="FILENAME",
+ type=argparse.FileType('w'), dest="outfile",
+ help="filename for the resulting sqlite database."
+ + " defaults to INPUT.sqlite")
+parser.add_argument("-i", "--ignore", metavar="FILENAME",
+ type=argparse.FileType('r'), dest="ignorefile",
+ help="filename for the ignores.")
+parser.add_argument('INPUT', nargs='?', action=readable_dir,
+ default=os.getcwd(),
+ help='input file or directory which is traversed'
+ + ' recursively. defaults to PWD')
args = parser.parse_args()
+# parse ignores
+ignores = parseIgnores(args.ignorefile) or []
+print ignores
# build output filename
if args.outfile== None:
- if os.path.isdir(args.INPUT):
- in_dir = os.path.abspath(args.INPUT)
- outfilename = in_dir + '/' + os.path.split(in_dir)[1] +
'.sqlite'
- elif os.path.isfile(args.INPUT):
- outfilename = os.path.splitext(args.INPUT)[0] + '.sqlite'
- else:
- print 'unknown output target, writing to expressions.sqlite'
- outfilename = 'expressions.sqlite'
+ if os.path.isdir(args.INPUT):
+ in_dir = os.path.abspath(args.INPUT)
+ tmp_name = os.path.split(in_dir)[1]
+ outfilename = in_dir + '/' + tmp_name + '.sqlite'
+ elif os.path.isfile(args.INPUT):
+ outfilename = os.path.splitext(args.INPUT)[0] + '.sqlite'
+ else:
+ print 'unknown output target, writing to expressions.sqlite'
+ outfilename = 'expressions.sqlite'
else:
- outfilename = args.outfile.name
- args.outfile.close()
+ outfilename = args.outfile.name
+ args.outfile.close()
# remove old db
try:
- os.remove(outfilename)
+ os.remove(outfilename)
except OSError:
- pass
+ pass
# connect to db
sql_conn = sqlite3.connect(outfilename)
@@ -55,113 +94,112 @@
# create table
sql_cur.execute('''CREATE TABLE Expression (
- expr_ID INTEGER PRIMARY KEY AUTOINCREMENT,
- file_name TEXT NOT NULL,
- expr_syntax TEXT,
- start_lineno INT,
- end_lineno INT,
- is_call INT,
- UNIQUE (file_name, expr_syntax, start_lineno,
end_lineno))''')
+ expr_ID INTEGER PRIMARY KEY AUTOINCREMENT,
+ file_name TEXT NOT NULL,
+ expr_syntax TEXT,
+ start_lineno INT,
+ end_lineno INT,
+ is_call INT,
+ UNIQUE (file_name, expr_syntax, start_lineno, end_lineno))''')
# parses one .sea file
def parse_sea_file(filename):
- scope_end = []
- scope_end.append(32767)
- parameters = deque()
- expr_list = []
- prevtype = ""
- seafile_lineno = 0
+ scope_end = []
+ scope_end.append(32767)
+ parameters = deque()
+ expr_list = []
+ prevtype = ""
+ seafile_lineno = 0
- try:
- infile = open(filename, 'r')
- print 'reading file: ',filename
- except IOError:
- print 'could not read file: ',filename
- infile.close()
- return
-
- cfilename = os.path.split(os.path.splitext(filename)[0])[1] + '.c'
+ try:
+ infile = open(filename, 'r')
+ print 'reading file: ',filename
+ except IOError:
+ print 'could not read file: ',filename
+ infile.close()
+ return
+ cfilename = os.path.split(os.path.splitext(filename)[0])[1] + '.c'
- lines = infile.readlines(10000)
- while lines:
- for line in lines:
- seafile_lineno += 1
- if prevtype == 'Begin-Expression':
- if re.match(re.escape("End-Expression: " +
rowfile + ':' + str(lineno) + ' ' + str(iscall)), line):
- vals = (rowfile, ' '.join(expr_list),
lineno, scope_end[-1], iscall)
- sql_cur.execute('INSERT OR IGNORE INTO
Expression VALUES (NULL,?,?,?,?,?)', vals)
- prevtype = 'End-Expression'
- expr_list.append(line.strip())
- continue
+ lines = infile.readlines(10000)
+ while lines:
+ for line in lines:
+ seafile_lineno += 1
+ if prevtype == 'Begin-Expression':
+ if re.match(re.escape("End-Expression: " + rowfile + ':' +
str(lineno) + ' ' + str(iscall)), line):
+ vals = (rowfile, ' '.join(expr_list), lineno,
scope_end[-1], iscall)
+ sql_cur.execute('INSERT OR IGNORE INTO Expression VALUES
(NULL,?,?,?,?,?)', vals)
+ prevtype = 'End-Expression'
+ expr_list.append(line.strip())
+ continue
- m = re.match(r"([A-Za-z-]+): ([^:]+):(\d+) (.+)", line)
- if not m or m.lastindex < 4:
- print 'unknown entry type: ', filename, ' on
line ', seafile_lineno
- prevtype = ''
- continue
- rowtype = m.group(1)
- rowfile = m.group(2)
+ m = re.match(r"([A-Za-z-]+): ([^:]+):(\d+) (.+)", line)
+ if not m or m.lastindex < 4:
+ print 'unknown entry type: ', filename, ' on line ',
seafile_lineno
+ prevtype = ''
+ continue
+ rowtype = m.group(1)
+ rowfile = m.group(2)
- if rowtype == 'SystemGlobal':
- globalvar = m.group(4)
- vals = (cfilename, globalvar, 0, 32767, 0)
- sql_cur.execute('INSERT OR IGNORE INTO
Expression VALUES (NULL,?,?,?,?,?)', vals)
- elif rowtype == 'Global':
- lineno = int(m.group(3))
- globalvar = m.group(4)
- vals = (rowfile, globalvar, lineno, 32767, 0)
- sql_cur.execute('INSERT OR IGNORE INTO
Expression VALUES (NULL,?,?,?,?,?)', vals)
- elif rowtype == 'Function':
- pass
-# print 'nothing to do yet for entry type
Function'
- elif rowtype == 'Parameter':
- if prevtype != 'Function' and prevtype !=
'Parameter':
- print 'Parameter can only follow
Function or another Parameter: ', filename, ' on line ', seafile_lineno
- lineno = int(m.group(3))
- para = m.group(4)
- parameters.append((rowfile, para, lineno))
- elif rowtype == 'Scope':
- scope_begin = int(m.group(3))
- scope_end.append(int(m.group(4)) + 1)
- while prevtype == 'Parameter':
- try:
- (rowfile, para, lineno) =
parameters.popleft()
- except IndexError:
- break
- vals = (rowfile, para, lineno,
scope_end[-1], 0)
- sql_cur.execute('INSERT OR IGNORE INTO
Expression VALUES (NULL,?,?,?,?,?)', vals)
- elif rowtype == 'Begin-Expression':
- lineno = int(m.group(3))
- iscall = int(m.group(4)[0])
- if lineno > scope_end[-1]:
- scope_end.pop()
- del expr_list[:]
- tmp = (m.group(4)[1:]).strip()
- if len(tmp) > 0: expr_list.append(tmp)
- else:
- print 'unknown entry type: ', filename, ' on
line ', seafile_lineno
- prevtype = rowtype
- sql_conn.commit()
- lines = infile.readlines(10000)
- infile.close()
+ if rowtype == 'SystemGlobal':
+ globalvar = m.group(4)
+ vals = (cfilename, globalvar, 0, 32767, 0)
+ sql_cur.execute('INSERT OR IGNORE INTO Expression VALUES
(NULL,?,?,?,?,?)', vals)
+ elif rowtype == 'Global':
+ lineno = int(m.group(3))
+ globalvar = m.group(4)
+ vals = (rowfile, globalvar, lineno, 32767, 0)
+ sql_cur.execute('INSERT OR IGNORE INTO Expression VALUES
(NULL,?,?,?,?,?)', vals)
+ elif rowtype == 'Function':
+ pass
+# print 'nothing to do yet for entry type Function'
+ elif rowtype == 'Parameter':
+ if prevtype != 'Function' and prevtype != 'Parameter':
+ print 'Parameter can only follow Function or another
Parameter: ', filename, ' on line ', seafile_lineno
+ lineno = int(m.group(3))
+ para = m.group(4)
+ parameters.append((rowfile, para, lineno))
+ elif rowtype == 'Scope':
+ scope_begin = int(m.group(3))
+ scope_end.append(int(m.group(4)) + 1)
+ while prevtype == 'Parameter':
+ try:
+ (rowfile, para, lineno) = parameters.popleft()
+ except IndexError:
+ break
+ vals = (rowfile, para, lineno, scope_end[-1], 0)
+ sql_cur.execute('INSERT OR IGNORE INTO Expression VALUES
(NULL,?,?,?,?,?)', vals)
+ elif rowtype == 'Begin-Expression':
+ lineno = int(m.group(3))
+ iscall = int(m.group(4)[0])
+ if lineno > scope_end[-1]:
+ scope_end.pop()
+ del expr_list[:]
+ tmp = (m.group(4)[1:]).strip()
+ if len(tmp) > 0: expr_list.append(tmp)
+ else:
+ print 'unknown entry type: ', filename, ' on line ',
seafile_lineno
+ prevtype = rowtype
+ sql_conn.commit()
+ lines = infile.readlines(10000)
+ infile.close()
# traverses all .sea files in the input folder
def traverse_sea_files(dummy, dirr, filess):
- for child in filess:
- if '.sea' == os.path.splitext(child)[1] and
os.path.isfile(dirr+'/'+child):
- parse_sea_file(dirr+'/'+child)
+ for child in filess:
+ if '.sea' == os.path.splitext(child)[1] and
os.path.isfile(dirr+'/'+child):
+ parse_sea_file(dirr+'/'+child)
# parse it
if os.path.isdir(args.INPUT):
- os.path.walk(os.path.abspath(args.INPUT), traverse_sea_files, 0)
+ os.path.walk(os.path.abspath(args.INPUT), traverse_sea_files, 0)
elif os.path.isfile(args.INPUT):
- parse_sea_file(args.INPUT)
+ parse_sea_file(args.INPUT)
else:
- print 'unknown INPUT, no rows written to ',outfilename
+ print 'unknown INPUT, no rows written to ',outfilename
# commit changes
sql_conn.commit()
| [Prev in Thread] |
Current Thread |
[Next in Thread] |
- [GNUnet-SVN] r25898 - monkey/trunk/seaspider,
gnunet <=