Merge lp:~jtv/corpusfiltergraph/pyflakes-cleanup into lp:~domy/corpusfiltergraph/old-sourceforge-trunk

[Jeroen T. Vermeulen]

Quick notes and cleanups based on pyflakes.

To prepare for an extraction of repeated code (the writing of status files), I ran all source files through the "pyflakes" code checker. It produced a large number of notes.

Here you see a few simple cleanups, but also many notes on undefined variables that look like they'd break the code. These may be in functions or code paths that aren't normally (or ever) used.

Jeroen

[PTTools Support (support-precisiontranslationtools)]

DIFF Line 48
- i = 0
I remember adding the i = 0 because there was some odd case where the following list comprehension failed with index out of range. This fixed it.

#TODO: "me" is undefined.
It's defined and it equates to self. See filtergraph.py Line 386. I'm sure it's a horrible way to do this, but I needed to share information with other classes:
  global me
  me = self

stats.py is a dated module that I think eventually became numpy. I use for only a few functions.

We'll need to work through others one-by-one.

[PTTools Support (support-precisiontranslationtools)]

DIFF Line 48
- i = 0
I remember adding the i = 0 because there was some odd case where the following list comprehension failed with index out of range. This fixed it.

#TODO: "me" is undefined.
It's defined and it equates to self. See filtergraph.py Line 386. I'm sure it's a horrible way to do this, but I needed to share information with other classes:
  global me
  me = self

stats.py is a dated module that I think eventually became numpy. I use for only a few functions.

We'll need to work through others one-by-one.

review: Approve

Unmerged revisions

340. By Jeroen T. Vermeulen on 2013-04-17

Superficial notes and cleanups based on pyflakes.

=== modified file 'trunk/lib/corpusfg/cfgparser.py'
--- trunk/lib/corpusfg/cfgparser.py	2013-01-01 15:15:07 +0000
+++ trunk/lib/corpusfg/cfgparser.py	2013-04-17 07:15:36 +0000
@@ -231,7 +231,7 @@
 
 	try:
 		reader = ','.join([v.strip() for v in [k for k in config.keys() if ',reader-' in k.replace(' ','')][0].split(',')])
-	except IndexError,e:
+	except IndexError:
 		errors.append([__name__,'missing','\"[reader]\" section not found'])
 	else:
 		if not 'stage' in config[reader]:
@@ -245,7 +245,7 @@
 		writer = ','.join([v.strip() for v in [k for k in config.keys() if ',writer-' in k.replace(' ','')][0].split(',')])
 		if not 'stage' in config[writer]:
 			errors.append([__name__,'missing','\"[%s] stage=\" value not found'%(writer)])
-	except IndexError,e:
+	except IndexError:
 		errors.append([__name__,'missing','\"[writer]\" section not found'])
 	else:
 		if not 'roottype' in config[writer]:
@@ -437,7 +437,8 @@
 					pass
 			return list(datetime), {} # make_lastupdate(master,datetime)
 		else:
-			return [ lines[0], cf.str2datetime(lines[0]) ], dict([[os.sep.join([path,a[0]]),a[1]] for a in [path.split('\t') for path in lines[1:]]])
+			# TODO: Two different "path" variables here.  Tease them apart.
+			return [ lines[0], cf.str2datetime(lines[0]) ], dict([[os.path.join(path,a[0]),a[1]] for a in [path.split('\t') for path in lines[1:]]])
 
 	# get date-time stamp from working folder
 	working_lastupdate, working_contents = get_lastupdate(working,epoch)
@@ -494,6 +495,7 @@
 		except:
 			raise OSError('Error: failed to open %s'%(outputini))
 		if os.name == 'posix' and os.geteuid() == 0:
+			# TODO: "working" is not defined!
 			cf.chmodrecurse(os.path.dirname(working),pwd.getpwnam(lrunas).pw_uid,pwd.getpwnam(lrunas).pw_gid)
 
 def _copytreeif(src, dst, datetime, updates={}, lastupdates={}, lrunas=''):

=== modified file 'trunk/lib/corpusfg/common.py'
--- trunk/lib/corpusfg/common.py	2013-01-01 15:15:07 +0000
+++ trunk/lib/corpusfg/common.py	2013-04-17 07:15:36 +0000
@@ -230,7 +230,6 @@
 	if listsize >= popsize: listsize = 1
 	tmplist = random.sample(xrange(popsize),listsize*nbrlists)
 	tmplen = int(len(tmplist)/nbrlists)
-	i = 0
 	return [tmplist[i:i+tmplen] for i in range(i,listsize*nbrlists,tmplen)]
 
 def samplesize(popsize,confidencelevel,confidenceinterval,estimate=0.5):
@@ -589,8 +588,12 @@
 	if os.name == 'nt':
 		try:
 			# note: Windows only allows admin users read the C:\windows\temp folder
-			temp = os.listdir(os.sep.join([os.environ.get('SystemRoot',''),'temp']))
+			# TODO: "temp" is unused.
+			temp = os.listdir(os.path.join(os.environ.get('SystemRoot',''),'temp'))
 		except:
+			# TODO: What kind of exception exactly?  Is the
+			# intention just to find out whether that temp directory
+			# exists?
 			return (os.environ['USERNAME'],False)
 		else:
 			return (os.environ['USERNAME'],True)
@@ -745,6 +748,4 @@
 at: http://www.precisiontranslationtools.com'''
 
 if __name__ == "__main__":
-	import os
-	import sys
 	sys.stdout.write(usage().encode('utf8')+'\n')

=== modified file 'trunk/lib/corpusfg/filtergraph.py'
--- trunk/lib/corpusfg/filtergraph.py	2013-01-01 15:15:07 +0000
+++ trunk/lib/corpusfg/filtergraph.py	2013-04-17 07:15:36 +0000
@@ -15,7 +15,6 @@
 import logging
 import flock
 import fnmatch
-import stat
 import common as cf
 import tempfile
 
@@ -31,6 +30,7 @@
 			try:
 				return super(filtergraph.CustomInput,self).__getitem__(key)
 			except KeyError:
+				# TODO: "me" is undefined.  Should it be "self"?
 				if me.stage[0].lower() == 'none':
 					filelist = me.filelist[:cf.stage]+list(key)
 				else:
@@ -72,6 +72,7 @@
 			try:
 				return super(filtergraph.CustomOutput,self).__getitem__(key)
 			except KeyError:
+				# TODO: "me" is undefined.  Should it be "self"?
 				fs,root = me.writer.getroot(me.writer.rootfolder) if me.writer.rootfolder else me.filelist[:cf.rttype]
 				if me.writer.stage.lower() == 'none':
 					filelist = [fs,root]+[me.writer.roottype]+list(key)
@@ -113,6 +114,7 @@
 			try:
 				return super(filtergraph.CustomError,self).__getitem__(key)
 			except KeyError:
+				# TODO: "me" is undefined.  Should it be "self"?
 				fs,root = me.writer.getroot(me.writer.rootfolder) if me.writer.rootfolder else me.filelist[:cf.rttype]
 				if me.writer.stage.lower() == 'none':
 					filelist = [fs,root]+[me.writer.roottype]+['-'.join([key[0],'workbench'])]+list(key[1:])
@@ -148,6 +150,7 @@
 			try:
 				return super(filtergraph.CustomExtract,self).__getitem__(key)
 			except KeyError:
+				# TODO: "me" is undefined.  Should it be "self"?
 				fs,root = me.writer.getroot(me.writer.rootfolder) if me.writer.rootfolder else me.filelist[:cf.rttype]
 				stage = me.filelist[cf.stage] if fnmatch.fnmatch(me.filelist[cf.stage],me.writer.stage) else me.writer.stage
 				filelist = [fs,root]+[me.writer.roottype]+['-'.join([stage,'extract'])]+list(key)
@@ -953,7 +956,7 @@
 
 		try:
 			alts = bool([k for k in self.cfoutput.keys() if not k[cf.kind].lower().startswith('tm') and not k[cf.kind].lower().startswith('lm')])
-		except Exception,e:
+		except Exception:
 			alts = True
 
 		if not alts:
@@ -1024,7 +1027,7 @@
 
 		try:
 			alts = bool([k for k in self.cfoutput.keys() if not k[cf.kind].lower().startswith('tm') and not k[cf.kind].lower().startswith('lm')])
-		except Exception,e:
+		except Exception:
 			alts = True
 
 		if not alts:

=== modified file 'trunk/lib/corpusfg/flock.py'
--- trunk/lib/corpusfg/flock.py	2013-01-01 15:15:07 +0000
+++ trunk/lib/corpusfg/flock.py	2013-04-17 07:15:36 +0000
@@ -9,6 +9,7 @@
 import os
 import socket
 
+# TODO: There's also a "from flock import flock" further down... Which is it!?
 class flock(object):
 	'''Class to handle creating and removing (pid) lockfiles'''
 

=== modified file 'trunk/lib/corpusfg/manager.py'
--- trunk/lib/corpusfg/manager.py	2013-01-01 15:15:07 +0000
+++ trunk/lib/corpusfg/manager.py	2013-04-17 07:15:36 +0000
@@ -8,12 +8,11 @@
 #version:
 #4.5.337 - version update
 
+import codecs
 import sys
 import os
 from flock import flock
-import codecs
 from progress_bar import ProgressBar
-import shutil
 import logging
 from filtergraph import filtergraph
 import common as cf
@@ -91,6 +90,7 @@
 			# create file handler which logs messages
 			self.logfile = 	os.path.abspath(os.path.expanduser(os.sep.join([cfg['filtergraph']['graphs'],cfg['filtergraph']['graphname'],os.extsep.join([cfg['filtergraph']['graphname'],'log'])])))
 			if cfg['debug'] and not sum([1 for handler in logger.handlers if 'baseFilename' in dir(handler) and handler.baseFilename == self.logfile]):
+				# TODO: Why re-import codecs!?
 				import codecs
 				import cfgparser
 				f = codecs.open(self.logfile,'w','utf8')
@@ -99,6 +99,7 @@
 				useropts = cfgparser.getuseroptions(inistring)
 				inistring = cfgparser.setuseroptions(inistring,useropts)
 				f.write('##### Parsed config.ini file##### :\x0a%s\x0a##################################\x0a\x0a\x0a'%(inistring))
+				# TODO: Why del(codecs)?
 				del(codecs)
 				del(cfgparser)
 				formatter = logging.Formatter("%(asctime)s\t%(levelname)s\t%(name)s\t%(lineno)s\t%(funcName)s()\t%(message)s")
@@ -430,7 +431,7 @@
 		try:
 			result = self.filtergraph.buildqueue(self.srcqueue,list(self.files))
 
-		except KeyboardInterrupt,e:
+		except KeyboardInterrupt:
 			logger.info('%s\t%s',*['stop','Ctrl-C user request'])
 			return 251
 

=== modified file 'trunk/lib/corpusfg/plugins/add-line-numbers.py'
--- trunk/lib/corpusfg/plugins/add-line-numbers.py	2013-01-01 15:15:07 +0000
+++ trunk/lib/corpusfg/plugins/add-line-numbers.py	2013-04-17 07:15:36 +0000
@@ -90,6 +90,7 @@
 		try:
 			# loop writes output line-by-line
 			for line in i:
+				# TODO: linenum is undefined.  Does this code work?
 				o.write('<s id=\"%d\">\t%s'%(linenum,line.rstrip('\r\n')))
 				progress.increment_amount()
 				sys.stderr.write(progress.bar)

=== modified file 'trunk/lib/corpusfg/plugins/aligner-passthrough.py'
--- trunk/lib/corpusfg/plugins/aligner-passthrough.py	2013-01-01 15:15:07 +0000
+++ trunk/lib/corpusfg/plugins/aligner-passthrough.py	2013-04-17 07:15:36 +0000
@@ -67,6 +67,5 @@
 at: http://www.precisiontranslationtools.com'''
 
 if __name__ == "__main__":
-	import os
 	import sys
 	sys.stdout.write(usage().encode('utf8')+'\n')

=== modified file 'trunk/lib/corpusfg/plugins/delete-files.py'
--- trunk/lib/corpusfg/plugins/delete-files.py	2013-01-01 15:15:07 +0000
+++ trunk/lib/corpusfg/plugins/delete-files.py	2013-04-17 07:15:36 +0000
@@ -11,7 +11,6 @@
 import os
 import sys
 import logging
-import shutil
 import common as cf
 
 logger = logging.getLogger('.'.join([os.path.splitext(os.path.basename(sys.argv[0]))[0],'manager','filtergraph',__name__]))

=== modified file 'trunk/lib/corpusfg/plugins/ja/convert-ansi2full.py'
--- trunk/lib/corpusfg/plugins/ja/convert-ansi2full.py	2013-01-01 15:15:07 +0000
+++ trunk/lib/corpusfg/plugins/ja/convert-ansi2full.py	2013-04-17 07:15:36 +0000
@@ -188,6 +188,7 @@
 
 			if self.exceptions:
 				for line in i:
+					# TODO: unicodedata is undefined.  Does this code work?
 					o.write('%s\n'%(u''.join([ch if ch in self.exception else unicodedata.normalize(self.form,ch) for ch in '\b'.join(line.rstrip('\r\n')).split('\b')])))
 					progress.increment_amount()
 					sys.stderr.write(progress.bar)

=== modified file 'trunk/lib/corpusfg/plugins/ja/convert-full2ansi.py'
--- trunk/lib/corpusfg/plugins/ja/convert-full2ansi.py	2013-01-01 15:15:07 +0000
+++ trunk/lib/corpusfg/plugins/ja/convert-full2ansi.py	2013-04-17 07:15:36 +0000
@@ -149,8 +149,6 @@
 	def close(self):
 		if skipclose: return
 
-		import os
-		import sys
 		import codecs
 		from progress_bar import ProgressBar
 
@@ -190,6 +188,7 @@
 
 			if self.exceptions:
 				for line in i:
+					# TODO: unicodedata is undefined.  Does this code work?
 					o.write('%s\n'%(u''.join([ch if ch in self.exception else unicodedata.normalize(self.form,ch) for ch in '\b'.join(line.rstrip('\r\n')).split('\b')])))
 					progress.increment_amount()
 					sys.stderr.write(progress.bar)

=== modified file 'trunk/lib/corpusfg/plugins/null-copyfile.py'
--- trunk/lib/corpusfg/plugins/null-copyfile.py	2013-01-01 15:15:07 +0000
+++ trunk/lib/corpusfg/plugins/null-copyfile.py	2013-04-17 07:15:36 +0000
@@ -10,7 +10,16 @@
 
 import os
 
-import common as cf
+# TODO: "logger" is used but undefined.  This program will probably crash
+# when trying to report anything out of the ordinary.
+
+# TODO: "filter" is not a good name for a class since there's already a
+# standard function of that name.
+
+# TODO: This is used as a global variable, but might be more appropriate as a
+# field inside the filter class.
+skipclose = False
+
 
 class filter(object):
 
@@ -98,6 +107,5 @@
 at: http://www.precisiontranslationtools.com'''
 
 if __name__ == "__main__":
-	import os
 	import sys
 	sys.stdout.write(usage().encode('utf8')+'\n')

=== modified file 'trunk/lib/corpusfg/plugins/reader-null.py'
--- trunk/lib/corpusfg/plugins/reader-null.py	2013-01-01 15:15:07 +0000
+++ trunk/lib/corpusfg/plugins/reader-null.py	2013-04-17 07:15:36 +0000
@@ -10,13 +10,10 @@
 
 import sys
 import os
-import stat
 import common as cf
 import logging
-import shutil
 import codecs
 from fnmatch import fnmatch
-import glob
 from progress_bar import ProgressBar
 
 logger = logging.getLogger('.'.join([os.path.splitext(os.path.basename(sys.argv[0]))[0],'manager','filtergraph',__name__]))

=== modified file 'trunk/lib/corpusfg/plugins/reader-tab.py'
--- trunk/lib/corpusfg/plugins/reader-tab.py	2013-01-01 15:15:07 +0000
+++ trunk/lib/corpusfg/plugins/reader-tab.py	2013-04-17 07:15:36 +0000
@@ -276,8 +276,5 @@
 at: http://www.precisiontranslationtools.com'''
 
 if __name__ == "__main__":
-	import os
-	import sys
-	import sys
 	sys.stdout.write(usage().encode('utf8')+'\n')
 	sys.stdout.write(usage().encode('utf8')+'\n')

=== modified file 'trunk/lib/corpusfg/plugins/reader-tmx.py'
--- trunk/lib/corpusfg/plugins/reader-tmx.py	2013-01-01 15:15:07 +0000
+++ trunk/lib/corpusfg/plugins/reader-tmx.py	2013-04-17 07:15:36 +0000
@@ -13,7 +13,6 @@
 import stat
 import common as cf
 import logging
-import shutil
 import codecs
 import xml.etree.cElementTree as ElementTree
 import re

=== modified file 'trunk/lib/corpusfg/plugins/remove-multi-spaces.py'
--- trunk/lib/corpusfg/plugins/remove-multi-spaces.py	2013-01-01 15:15:07 +0000
+++ trunk/lib/corpusfg/plugins/remove-multi-spaces.py	2013-04-17 07:15:36 +0000
@@ -14,6 +14,8 @@
 import common as cf
 import logging
 
+from progress_bar import ProgressBar
+
 logger = logging.getLogger('.'.join([os.path.splitext(os.path.basename(sys.argv[0]))[0],'manager','filtergraph',__name__]))
 skipclose = True
 

=== modified file 'trunk/lib/corpusfg/plugins/repair-escapes.py'
--- trunk/lib/corpusfg/plugins/repair-escapes.py	2013-01-01 15:15:07 +0000
+++ trunk/lib/corpusfg/plugins/repair-escapes.py	2013-04-17 07:15:36 +0000
@@ -12,7 +12,6 @@
 import os
 import sys
 import re
-import htmlentitydefs
 import common as cf
 import logging
 

=== modified file 'trunk/lib/corpusfg/plugins/special-join.py'
--- trunk/lib/corpusfg/plugins/special-join.py	2013-01-01 15:15:07 +0000
+++ trunk/lib/corpusfg/plugins/special-join.py	2013-04-17 07:15:36 +0000
@@ -9,7 +9,6 @@
 #4.5.337 - version update
 
 from copy import deepcopy
-import common as cf
 
 class filter(object):
 

=== modified file 'trunk/lib/corpusfg/plugins/special-secondary-output.py'
--- trunk/lib/corpusfg/plugins/special-secondary-output.py	2013-01-01 15:15:07 +0000
+++ trunk/lib/corpusfg/plugins/special-secondary-output.py	2013-04-17 07:15:36 +0000
@@ -53,7 +53,8 @@
 		self.cfextract[knew]['tempbuff'] = list(self.p.cfoutput[k]['tempbuff'])
 		try:
 			codecs.open(self.cfextract[knew]['tempname'],self.append,self.encoding).write(self.eol.join(self.cfextract[knew]['tempbuff'])+self.eol)
-		except Exception,e:
+		except Exception:
+			# TODO: Log the exception to help debugging.
 			return -2
 		else:
 			ext = self.p.writer.rendertypes

=== modified file 'trunk/lib/corpusfg/plugins/special-split.py'
--- trunk/lib/corpusfg/plugins/special-split.py	2013-01-01 15:15:07 +0000
+++ trunk/lib/corpusfg/plugins/special-split.py	2013-04-17 07:15:36 +0000
@@ -9,7 +9,6 @@
 #4.5.337 - version update
 
 from copy import deepcopy
-import common as cf
 
 class filter(object):
 

=== modified file 'trunk/lib/corpusfg/plugins/subprocess-synch.py'
--- trunk/lib/corpusfg/plugins/subprocess-synch.py	2013-01-01 15:15:07 +0000
+++ trunk/lib/corpusfg/plugins/subprocess-synch.py	2013-04-17 07:15:36 +0000
@@ -146,6 +146,7 @@
 		try:
 			# loop writes output line-by-line
 			for line in i:
+				# TODO: tgtlang is undefined.  Does this code work?
 				subproc[tgtlang]['popen'].stdin.write('%s\n'%(line.encode(self.encoding).strip()))
 				o.write('%s\n'%(subproc[tgtlang]['popen'].stdout.readline().rstrip('\r\n').decode(self.encoding)))
 				progress.increment_amount()

=== modified file 'trunk/lib/corpusfg/plugins/writer-file.py'
--- trunk/lib/corpusfg/plugins/writer-file.py	2013-01-01 15:15:07 +0000
+++ trunk/lib/corpusfg/plugins/writer-file.py	2013-04-17 07:15:36 +0000
@@ -13,7 +13,6 @@
 import shutil
 import codecs
 from fnmatch import fnmatch
-from flock import flock
 import logging
 import common as cf
 

=== modified file 'trunk/lib/corpusfg/plugins/writer-null.py'
--- trunk/lib/corpusfg/plugins/writer-null.py	2013-01-01 15:15:07 +0000
+++ trunk/lib/corpusfg/plugins/writer-null.py	2013-04-17 07:15:36 +0000
@@ -10,12 +10,8 @@
 
 import sys
 import os
-import shutil
-import codecs
 from fnmatch import fnmatch
-from flock import flock
 import logging
-import common as cf
 
 logger = logging.getLogger('.'.join([os.path.splitext(os.path.basename(sys.argv[0]))[0],'manager','filtergraph',__name__]))
 

=== modified file 'trunk/lib/corpusfg/plugins/writer-tab.py'
--- trunk/lib/corpusfg/plugins/writer-tab.py	2013-01-01 15:15:07 +0000
+++ trunk/lib/corpusfg/plugins/writer-tab.py	2013-04-17 07:15:36 +0000
@@ -13,7 +13,6 @@
 import shutil
 import codecs
 from fnmatch import fnmatch
-from flock import flock
 import logging
 import common as cf
 

=== modified file 'trunk/lib/corpusfg/plugins/writer-tmx.py'
--- trunk/lib/corpusfg/plugins/writer-tmx.py	2013-01-01 15:15:07 +0000
+++ trunk/lib/corpusfg/plugins/writer-tmx.py	2013-04-17 07:15:36 +0000
@@ -11,9 +11,7 @@
 import sys
 import os
 import shutil
-import re
 from fnmatch import fnmatch
-from flock import flock
 import xml.etree.cElementTree as ElementTree
 from StringIO import StringIO
 import logging

=== modified file 'trunk/lib/corpusfg/plugins/zh_cn/convert-zh2traditional.py'
--- trunk/lib/corpusfg/plugins/zh_cn/convert-zh2traditional.py	2013-01-01 15:15:07 +0000
+++ trunk/lib/corpusfg/plugins/zh_cn/convert-zh2traditional.py	2013-04-17 07:15:36 +0000
@@ -116,6 +116,7 @@
 
 			if self.exceptions:
 				for line in i:
+					# TODO: unicodedata is undefined.  Does this code work?
 					o.write('%s\n'%(u''.join([ch if ch in self.exception else unicodedata.normalize(self.form,ch) for ch in '\b'.join(line.rstrip('\r\n')).split('\b')])))
 					progress.increment_amount()
 					sys.stderr.write(progress.bar)

=== modified file 'trunk/lib/corpusfg/plugins/zh_hk/convert-zh2simplified.py'
--- trunk/lib/corpusfg/plugins/zh_hk/convert-zh2simplified.py	2013-01-01 15:15:07 +0000
+++ trunk/lib/corpusfg/plugins/zh_hk/convert-zh2simplified.py	2013-04-17 07:15:36 +0000
@@ -116,6 +116,7 @@
 
 			if self.exceptions:
 				for line in i:
+					# TODO: unicodedata is undefined.  Does this code work?
 					o.write('%s\n'%(u''.join([ch if ch in self.exception else unicodedata.normalize(self.form,ch) for ch in '\b'.join(line.rstrip('\r\n')).split('\b')])))
 					progress.increment_amount()
 					sys.stderr.write(progress.bar)

=== modified file 'trunk/lib/corpusfg/progress_bar.py'
--- trunk/lib/corpusfg/progress_bar.py	2013-01-01 15:15:07 +0000
+++ trunk/lib/corpusfg/progress_bar.py	2013-04-17 07:15:36 +0000
@@ -42,7 +42,7 @@
 along with this program.  If not, see http://www.gnu.org/licenses/.'''
 
 
-import sys, os
+import sys
  
 class ProgressBar:
 

=== modified file 'trunk/lib/corpusfg/pstat.py'
--- trunk/lib/corpusfg/pstat.py	2012-05-02 15:46:23 +0000
+++ trunk/lib/corpusfg/pstat.py	2013-04-17 07:15:36 +0000
@@ -108,8 +108,10 @@
 ##
 ## 11/08/98 ... fixed aput to output large arrays correctly
 
+# TODO: Why is stats required?  It's not used in here.
 import stats  # required 3rd party module
 import string, copy
+# TODO: Is this import needed?
 from types import *
 
 __version__ = 0.4
@@ -120,6 +122,7 @@
 ### Array functions (for NumPy-enabled computers) appear below.
 ###
 
+# TODO: abut?  Or about?  Is this actually used?
 def abut (source,*args):
 	'''
 Like the |Stat abut command.  It concatenates two lists side-by-side

=== modified file 'trunk/lib/corpusfg/stats.py'
--- trunk/lib/corpusfg/stats.py	2012-05-02 15:46:23 +0000
+++ trunk/lib/corpusfg/stats.py	2013-04-17 07:15:36 +0000
@@ -227,6 +227,7 @@
 
 import pstat				# required 3rd party module
 import math, string, copy	# required python modules
+# TODO: Is this import needed?
 from types import *
 
 __version__ = 0.6
@@ -420,7 +421,7 @@
 	'''
 	try:
 		x = moment(inlist,3)/pow(moment(inlist,2),1.5)
-	except Exception, e:
+	except Exception:
 		x = 0
 	return x
 
@@ -432,9 +433,10 @@
 
 	Usage:  lkurtosis(inlist)
 	'''
+	# TODO: This always returns 0.  Is there a point?
 	try:
 		x = moment(inlist,4)/pow(moment(inlist,2),2.0)
-	except Exception, e:
+	except Exception:
 		x = 0
 	return 0
 
@@ -677,7 +679,7 @@
 		deviations[i] = inlist[i] - mn
 	try:
 		x = ss(deviations)/float(n-1)
-	except Exception, e:
+	except Exception:
 		x = 0
 	return x
 
@@ -859,6 +861,7 @@
 	n = len(x)
 	x = map(float,x)
 	y = map(float,y)
+	# TODO: xmean and ymean are not used.  Are they needed?
 	xmean = mean(x)
 	ymean = mean(y)
 	r_num = n*(summult(x,y)) - sum(x)*sum(y)
@@ -892,6 +895,7 @@
 	Usage:  lspearmanr(x,y) where x and y are equal-length lists
 	Returns: Spearman's r, two-tailed p-value
 	'''
+	# TODO: TINY is not used.  Is it needed?
 	TINY = 1e-30
 	if len(x) <> len(y):
 		raise ValueError, 'Input values not paired in spearmanr. Aborting.'
@@ -927,6 +931,7 @@
 		raise ValueError, "Exactly 2 categories required for pointbiserialr()."
 	else:				# there are 2 categories, continue
 		codemap = pstat.abut(categories,range(2))
+		# TODO: recoded is not used.  Is it needed?
 		recoded = pstat.recode(data,codemap,0)
 		x = pstat.linexand(data,0,categories[0])
 		y = pstat.linexand(data,0,categories[1])
@@ -994,6 +999,7 @@
 	r_num = float(n*(summult(x,y)) - sum(x)*sum(y))
 	r_den = math.sqrt((n*ss(x) - square_of_sums(x))*(n*ss(y)-square_of_sums(y)))
 	r = r_num / r_den
+	# TODO: z is not used.  Is it needed?
 	z = 0.5*math.log((1.0+r+TINY)/(1.0-r+TINY))
 	df = n-2
 	t = r*math.sqrt(df/((1.0-r+TINY)*(1.0+r+TINY)))
@@ -1175,6 +1181,7 @@
 	n2 = len(y)
 	ranked = rankdata(x+y)
 	rankx = ranked[0:n1]						# get the x-ranks
+	# TODO: ranky is not used.  Is it needed?
 	ranky = ranked[n1:]							# the rest are y-ranks
 	u1 = n1*n2 + (n1*(n1+1))/2.0 - sum(rankx)	# calc U for x
 	u2 = n1*n2 - u1								# remainder is U for y
@@ -1582,8 +1589,11 @@
 	ns = [0]*a
 	alldata = []
 	tmp = map(N.array,lists)
+	# TODO: means is not used.  Is it needed?
 	means = map(amean,tmp)
+	# TODO: Assigning to vars is weird and scary.  Can we avoid it?
 	vars = map(avar,tmp)
+	# TODO: ns is not used.  Is it needed?
 	ns = map(len,lists)
 	for i in range(len(lists)):
 		alldata = alldata + lists[i]
@@ -1823,6 +1833,7 @@
 	'''
 	suffix = ''							# for *s after the p-value
 	try:
+		# TODO: x is not used.  Is it needed?
 		x = prob.shape
 		prob = prob[0]
 	except:
@@ -2037,6 +2048,7 @@
 			if keepdims == 1:
 				shp = list(inarray.shape)
 				shp[dimension] = 1
+				# TODO: Is this meant to override sum()!?
 				sum = N.reshape(sum,shp)
 		else: # must be a SEQUENCE of dims to average over
 			dims = list(dimension)
@@ -2371,6 +2383,7 @@
 			mask = lowerfcn(a,limits[0])
 		elif limits[0]<>None and limits[1]<>None:
 			mask = lowerfcn(a,limits[0])*upperfcn(a,limits[1])
+		# TODO: term1 is not used.  Is it needed?
 		term1 = N.add.reduce(N.ravel(a*a*mask))
 		n = float(N.add.reduce(N.ravel(mask)))
 		return sd/math.sqrt(n)
@@ -2468,6 +2481,7 @@
 		mm = (N.minimum.reduce(inarray),N.maximum.reduce(inarray))
 		m = amean(inarray,dimension)
 		sd = astdev(inarray,dimension)
+		# TODO: skew is not used.  Is it needed?
 		skew = askew(inarray,dimension)
 		kurt = akurtosis(inarray,dimension)
 #		return n, mm, m, sd, skew, kurt
@@ -3137,6 +3151,7 @@
 		'''
 		TINY = 1.0e-20
 		n = len(x)
+		# TODO: xmean and ymean are not used.  Are they needed?
 		xmean = amean(x)
 		ymean = amean(y)
 		r_num = n*(N.add.reduce(x*y)) - N.add.reduce(x)*N.add.reduce(y)
@@ -3156,6 +3171,7 @@
 		Usage:  aspearmanr(x,y) where x,y are equal-length arrays
 		Returns: Spearman's r, two-tailed p-value
 		'''
+		# TODO: TINY is not used.  Is it needed?
 		TINY = 1e-30
 		n = len(x)
 		rankx = rankdata(x)
@@ -3186,6 +3202,7 @@
 			raise ValueError, "Exactly 2 categories required (in x) for pointbiserialr()."
 		else:							# there are 2 categories, continue
 			codemap = pstat.aabut(categories,N.arange(2))
+			# TODO: recoded is not used.  Is it needed?
 			recoded = pstat.arecode(data,codemap,0)
 			x = pstat.alinexand(data,0,categories[0])
 			y = pstat.alinexand(data,0,categories[1])
@@ -3262,6 +3279,7 @@
 		r_num = n*(N.add.reduce(x*y)) - N.add.reduce(x)*N.add.reduce(y)
 		r_den = math.sqrt((n*ass(x) - asquare_of_sums(x))*(n*ass(y)-asquare_of_sums(y)))
 		r = r_num / r_den
+		# TODO: z is not used.  Is it needed?
 		z = 0.5*math.log((1.0+r+TINY)/(1.0-r+TINY))
 		df = n-2
 		t = r*math.sqrt(df/((1.0-r+TINY)*(1.0+r+TINY)))
@@ -3304,6 +3322,7 @@
 		r_den = N.where(zerodivproblem,1,r_den)			# avoid zero-division in 1st place
 		r = r_num / r_den								# need to do this nicely for matrix division
 		r = N.where(zerodivproblem,0.0,r)
+		# TODO: z is not used.  Is it needed?
 		z = 0.5*N.log((1.0+r+TINY)/(1.0-r+TINY))
 		df = n-2
 		t = r*N.sqrt(df/((1.0-r+TINY)*(1.0+r+TINY)))
@@ -3555,6 +3574,7 @@
 		n2 = len(y)
 		ranked = rankdata(N.concatenate((x,y)))
 		rankx = ranked[0:n1]						# get the x-ranks
+		# TODO: ranky is not used.  Is it needed?
 		ranky = ranked[n1:]							# the rest are y-ranks
 		u1 = n1*n2 + (n1*(n1+1))/2.0 - sum(rankx)	# calc U for x
 		u2 = n1*n2 - u1								# remainder is U for y
@@ -3725,6 +3745,7 @@
 			exponents = N.where(N.less(x,-BIG),-BIG,x)
 			return N.exp(exponents)
 
+		# TODO: arrayflag is not used.  Is it needed?
 		if type(chisq) == N.ndarray:
 			arrayflag = 1
 		else:
@@ -4017,7 +4038,7 @@
 #######  AANOVA CALCULATIONS  #######
 #####################################
 
-	import LinearAlgebra, operator
+	import LinearAlgebra
 	LA = LinearAlgebra
 
 	def aglm(data,para):
@@ -4069,8 +4090,11 @@
 		ns = [0]*na
 		alldata = []
 		tmp = map(N.array,args)
+		# TODO: means is never used.  Is it needed?
 		means = map(amean,tmp)
+		# TODO: Assigning to vars is weird and scary.  Can we avoid it?
 		vars = map(avar,tmp)
+		# TODO: ns is never used.  Is it needed?
 		ns = map(len,args)
 		alldata = N.concatenate(args)
 		bign = len(alldata)
@@ -4281,6 +4305,7 @@
 		Returns: sum[ravel(a-b)**2]
 		'''
 		if dimension == None:
+			# TODO: inarray is never used.  Is it needed?
 			inarray = N.ravel(a)
 			dimension = 0
 		return asum((a-b)**2,dimension,keepdims)

=== modified file 'trunk/lib/corpusfg/unicode_block.py'
--- trunk/lib/corpusfg/unicode_block.py	2012-05-02 15:46:23 +0000
+++ trunk/lib/corpusfg/unicode_block.py	2013-04-17 07:15:36 +0000
@@ -6,6 +6,7 @@
 	if type(ch) == type(''): ch = unicode(ch)
 	assert isinstance(ch, unicode) and len(ch) == 1, repr(ch)
 	cp = ord(ch)
+	# TODO: _blocks is undefined.  Is this function used?
 	for start, end, name in _blocks:
 		if start <= cp <= end:
 			return unicode(name)