Merge lp:~gt-launchpad/ladon/test into lp:ladon

[gthomas]

overwritten from ~ladon-dev-team/ladon/ladon and took those soap xml changes.

[lp:~gt-launchpad/ladon/test]

42. By Tamás Gulácsi <email address hidden> on 2011-08-20

add test for generator types (list of dicts)

43. By Tamás Gulácsi <email address hidden> on 2011-08-21

trying send generators

44. By Tamás Gulácsi <email address hidden> on 2011-08-21

jsonwsp: let generators pass through

45. By Tamás Gulácsi <email address hidden> on 2011-08-22

merged upstream

46. By Tamás Gulácsi <email address hidden> on 2011-08-26

merge upstream

47. By Tamás Gulácsi <email address hidden> on 2011-08-26

get rid of "exec" calls

48. By Tamás Gulácsi <email address hidden> on 2011-08-28

client/jsonwsp get-rid-of-exec, part II.

49. By Tamás Gulácsi <email address hidden> on 2011-08-28

generator simple test case works for jsonwsp

50. By Tamás Gulácsi <email address hidden> on 2011-08-29

merge upstream

51. By Tamás Gulácsi <email address hidden> on 2011-08-30

Merge upstream + add profiling capability to testladon.py + add iaggregate to wsgi

52. By Tamás Gulácsi <email address hidden> on 2011-09-25

merge upstream

53. By Tamás Gulácsi <email address hidden> on 2012-03-03

merged upstream

Unmerged revisions

53. By Tamás Gulácsi <email address hidden> on 2012-03-03

merged upstream

52. By Tamás Gulácsi <email address hidden> on 2011-09-25

merge upstream

51. By Tamás Gulácsi <email address hidden> on 2011-08-30

Merge upstream + add profiling capability to testladon.py + add iaggregate to wsgi

50. By Tamás Gulácsi <email address hidden> on 2011-08-29

merge upstream

49. By Tamás Gulácsi <email address hidden> on 2011-08-28

generator simple test case works for jsonwsp

48. By Tamás Gulácsi <email address hidden> on 2011-08-28

client/jsonwsp get-rid-of-exec, part II.

47. By Tamás Gulácsi <email address hidden> on 2011-08-26

get rid of "exec" calls

46. By Tamás Gulácsi <email address hidden> on 2011-08-26

merge upstream

45. By Tamás Gulácsi <email address hidden> on 2011-08-22

merged upstream

44. By Tamás Gulácsi <email address hidden> on 2011-08-21

jsonwsp: let generators pass through

=== modified file 'frameworks/python/src/ladon/ladonizer/decorator.py'
--- frameworks/python/src/ladon/ladonizer/decorator.py	2011-07-24 22:08:25 +0000
+++ frameworks/python/src/ladon/ladonizer/decorator.py	2012-03-03 20:32:18 +0000
@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+# vim: set noet:
 
 """
 This module contains the entry-point for all Ladon Service methods.
@@ -22,12 +23,12 @@
 	The ladonize decorator takes exactly the same amount of
 	arguments as the method it is decorating plus one mandatory
 	keyword argument *rtype* that type-defines the return type of
-	the method. 
+	the method.
 	Each argument given to the decorator type-defines the order-
 	wise corresponding parameter of the method being decorated::
-		
+
 		class SessionService(object):
-		
+
 			@ladonize(str,str,rtype=str)
 			def login(self,username,password):
 				...
@@ -52,15 +53,15 @@
 			user method is called. The result of the userspace-method is then checked before it is
 			passed back to the dispatcher.
 			"""
-			
+
 			# Get the LadonMethodInfo object generated at parsetime which is stored as a member
 			# on the function object
 			lmi = injector._ladon_method_info
 			# Reference the incomming arguments in callargs (filter out the function reference)
 			callargs = args[1:]
-			
+
 			for argidx in range(len(callargs)):
-				# Check the type of each argument against the type which the method has been 
+				# Check the type of each argument against the type which the method has been
 				# registered to take in the order-wise corresponding argument
 				if not validate_type(lmi._arg_types[argidx],callargs[argidx]):
 					# Raise ArgTypeMismatch
@@ -70,27 +71,27 @@
 						lmi._arg_names[argidx],
 						lmi._arg_types[argidx],
 						type(callargs[argidx]))
-			
+
 			# Call the userspace service method (**kw will be used to transport Ladon info
 			# and tools all the way to userspace of the service method. I.e. the TypeConverter
 			# is passed with the keyword "LADON_METHOD_TC")
 			res = f(*args,**kw)
-			
+
 			# Check the return type
 			if not validate_type(lmi._rtype,res):
-				# Raise Arg-type mismatch 
+				# Raise Arg-type mismatch
 				raise ReturnTypeMismatch(lmi.sinfo,lmi._func_name,lmi._rtype,type(res))
-			
+
 			# Return the result to the dispatcher
 			return res
-		
-		# Register the service method and all the types required by it 
+
+		# Register the service method and all the types required by it
 		ladon_method_info = global_service_collection().add_service_method(f,*def_args,**def_kw)
-		
+
 		# store the LadonMethodInfo object directly on the fuction object
 		injector._ladon_method_info = ladon_method_info
 		injector.__doc__ = ladon_method_info._doc
 		injector.func_name = ladon_method_info._func_name
 		return injector
-		
-	return decorator 
+
+	return decorator

=== added file 'frameworks/python/src/ladon/pf'
--- frameworks/python/src/ladon/pf	1970-01-01 00:00:00 +0000
+++ frameworks/python/src/ladon/pf	2012-03-03 20:32:18 +0000
@@ -0,0 +1,1 @@
+exec pyflake --ignore=W191,E225,E303,E302,E231,E501,E202,E501,W391 "$@"

=== added file 'frameworks/python/src/ladon/tools/iterencode.py'
--- frameworks/python/src/ladon/tools/iterencode.py	1970-01-01 00:00:00 +0000
+++ frameworks/python/src/ladon/tools/iterencode.py	2012-03-03 20:32:18 +0000
@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+# coding: utf-8
+# vim: set noet:
+
+import logging
+LOG = logging.getLogger(__name__)
+import json
+import types
+
+list_types = (tuple, list, types.GeneratorType)
+
+class GeneratorAsList(list):
+	def __init__(self, generator):
+		self._generator = generator
+
+	def __nonzero__(self):
+		return bool(self._generator)
+
+	def __iter__(self):
+		return self._generator
+
+	def __len__(self):
+		raise NotImplementedError
+
+class JSONIterEncoder(json.JSONEncoder):
+	def default(self, obj):
+		#LOG.debug('default: %r %s', obj, type(obj))
+		if isinstance(obj, list_types):
+			return GeneratorAsList(obj)
+		else:
+			super(self.__class__, self).default(obj)
+
+
+def iaggregate(iterobj, chunksize=8192):
+    '''aggregates iter chunks to the given size'''
+    n = 0
+    data = []
+    for chunk in iterobj:
+        data.append(chunk)
+        n += len(chunk)
+        if n >= chunksize:
+            #print repr(data)
+            yield ''.join(data)
+            data = []
+            n = 0
+    #
+    if data:
+        yield ''.join(data)
+

=== modified file 'frameworks/python/src/ladon/types/__init__.py'
--- frameworks/python/src/ladon/types/__init__.py	2011-07-11 09:37:30 +0000
+++ frameworks/python/src/ladon/types/__init__.py	2012-03-03 20:32:18 +0000
@@ -1,16 +1,39 @@
 # -*- coding: utf-8 -*-
+# vim: se noet:
+
+import types
+list_types = (tuple, list, types.GeneratorType)
+import logging
+LOG = logging.getLogger(__name__)
+import itertools
+
+
+def generator_peek(generator):
+	'''returns a (first_item, full_generator) tuple'''
+	try:
+		first = generator.next()
+	except StopIteration:
+		return (None, [])
+	else:
+		return (first, itertools.chain([first], generator))
+
 
 def validate_type(typ,val):
-	if [tuple,list].count(type(typ)):
-		if not [tuple,list].count(type(val)):
-			return False
-		for i in val:
-			if type(i)!=typ[0]:
-				return False
+	# LOG.debug('validate_type(%r, %s)', typ, type(val))
+	ok = False
+	if isinstance(typ, list_types):
+		# LOG.debug('val=%r %s %r', val, isinstance(val, list_types), typ[0])
+		if isinstance(val, (tuple, list)):
+			ok = all(isinstance(i, typ[0]) for i in val)
+		else:
+			first, val = generator_peek(val)
+			ok = not val or isinstance(first, typ[0])
 	else:
-		if typ!=type(val):
-			return False
-	return True
+		ok = isinstance(val, typ)
+	if not ok:
+		LOG.warn('%s is NOT %r', type(val), typ)
+	return ok
+
 
 def get_type_info(typ):
 	"""
@@ -18,7 +41,9 @@
 	the type dict will be returned, otherwise None is returned.
 	"""
 	from ladon.types.typemanager import TypeManager
-	if not [list,tuple].count(type(typ)) and typ in TypeManager.global_type_dict:
-		return TypeManager.global_type_dict[typ]
-	else:
-		return None
+	info = None
+	if not isinstance(typ, list_types) and typ in TypeManager.global_type_dict:
+		info = TypeManager.global_type_dict[typ]
+	return info
+
+__all__ = ['validate_type', 'get_type_info', 'list_types', 'generator_peek']

=== modified file 'frameworks/python/tests/servicerunner.py'
--- frameworks/python/tests/servicerunner.py	2011-07-11 09:37:30 +0000
+++ frameworks/python/tests/servicerunner.py	2012-03-03 20:32:18 +0000
@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+# vim: set noet:
 
 from ladon.server.wsgi import LadonWSGIApplication
 import wsgiref.simple_server
@@ -34,6 +35,10 @@
 		server = make_server()
 		server.serve_forever()
 	return None
-    
+
 if __name__=='__main__':
+	import sys
+	import logging
+	logging.basicConfig(level=logging.DEBUG if '-v' in sys.argv[1:]
+			else logging.INFO)
 	serve_test_service()

=== modified file 'frameworks/python/tests/services/typetests.py'
--- frameworks/python/tests/services/typetests.py	2011-07-11 09:37:30 +0000
+++ frameworks/python/tests/services/typetests.py	2012-03-03 20:32:18 +0000
@@ -1,6 +1,9 @@
 # -*- coding: utf-8 -*-
+# vim: set noet:
 from ladon.ladonizer import ladonize
 from ladon.compat import PORTABLE_BYTES,PORTABLE_STRING,PORTABLE_STRING_TYPES
+import types
+from ladon.types.ladontype import LadonType
 # Portable types are defined like this in ladon.compat:
 #
 #if sys.version_info[0]==2:
@@ -12,9 +15,32 @@
 	#PORTABLE_STRING = str
 	#PORTABLE_STRING_TYPES = [str,bytes]
 
+c_keys = ['%s_%02d' % (k, i) for i in range(20) for k in ('bytes', 'str')]
+
+
+class Record(LadonType):
+	__slots__ = c_keys
+
+	locals().update(dict((k,
+		 PORTABLE_BYTES if k.startswith('bytes_') else PORTABLE_STRING)
+			 for k in c_keys))
+
+	def __init__(self, adict={}):
+		for k, v in adict.iteritems():
+			setattr(self, k, v)
+
+
 class TypeTestService(object):
-	
+
 	@ladonize(PORTABLE_BYTES,rtype=PORTABLE_BYTES,encoding='utf-8')
 	def conversion(self,in_bytes):
 		in_str = in_bytes.decode('utf-8')
 		return in_str.encode('utf-8')
+
+	@ladonize(PORTABLE_BYTES, int, rtype=[Record], encoding='utf-8')
+	def gen_dicts(self, in_bytes, count=1000):
+		import itertools
+		in_str = in_bytes.decode('utf-8')
+		values = itertools.cycle((in_bytes, in_str))
+		rec = Record(dict((k, values.next()) for k in c_keys))
+		return (rec for _ in xrange(count))

=== modified file 'frameworks/python/tests/testladon.py'
--- frameworks/python/tests/testladon.py	2011-07-11 09:37:30 +0000
+++ frameworks/python/tests/testladon.py	2012-03-03 20:32:18 +0000
@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+# vim: set noet:
 
 import unittest
 import servicerunner
@@ -14,6 +15,9 @@
 import sys,json
 from ladon.compat import PORTABLE_BYTES,PORTABLE_STRING,PORTABLE_STRING_TYPES
 
+import logging
+LOG = logging.getLogger(__name__)
+
 def str_to_portable_string(in_str):
 	"""
 	Assumes that we always use UTF-8 encoding in script files
@@ -25,7 +29,7 @@
 
 
 class HTTPRequestPoster(object):
-	
+
 	def __init__(self,url):
 		self.valid_url = True
 		parseres = urlparse(url)
@@ -40,7 +44,7 @@
 		if str(custom_port).isdigit():
 			self.port = int(custom_port)
 		self.path = parseres.path
-	
+
 	def post_request(self,data,extra_path="jsonwsp",encoding="utf-8"):
 		headers = {
 			"Content-type": "application/json, charset=%s" % encoding,
@@ -63,29 +67,30 @@
 	def setUp(self):
 		self.post_helper = HTTPRequestPoster('http://localhost:2376/StringTestService')
 
-	def string_integrety_tests_json(self,methodname):
+	def string_integrity_tests_json(self,methodname):
 		fp_req = open('data/stringtests/in_out_test.json','rb')
 		req = PORTABLE_STRING(fp_req.read(),'utf-8')
 		fp_req.close()
 		req = json.loads(req)
 		req['methodname'] = methodname
 		req = json.dumps(req)
-		
+
 		# utf-8 encoded request to bytesEncodingTest
 		status,reason,resdata = self.post_helper.post_request(req.encode('utf-8'),encoding='utf-8')
-		self.assertEqual(status, 200)
+		self.assertEqual(status, 200, resdata)
+		LOG.debug('resdata: %r', resdata)
 		res = json.loads(PORTABLE_STRING(resdata,'utf-8'))
 		expected_result = str_to_portable_string('äöüÄÖÄæøåÆØÅß')
 		self.assertEqual(res['result'], expected_result)
-		
+
 		# latin-1 encoded request to bytesEncodingTest
 		status,reason,resdata = self.post_helper.post_request(req.encode('latin-1'),encoding='latin-1')
-		self.assertEqual(status, 200)
+		self.assertEqual(status, 200, resdata)
 		res = json.loads(PORTABLE_STRING(resdata,'latin-1'))
 		self.assertEqual(res['result'], expected_result)
 
 
-	def string_integrety_tests_soap(self,methodname):
+	def string_integrity_tests_soap(self,methodname,mangler=None):
 		fp_req = open('data/stringtests/in_out_test.soap','rb')
 		req = fp_req.read()
 		fp_req.close()
@@ -94,50 +99,114 @@
 
 		expected_result = str_to_portable_string('äöüÄÖÄæøåÆØÅß')
 
+		def toxml(xml, encoding='utf-8'):
+			text = xml.toxml(encoding=encoding)
+			if mangler:
+				text = mangler(text)
+			LOG.debug('toxml: %r\n%s', text, text)
+			return text
 		# utf-8 encoded request to bytesEncodingTest
-		status,reason,resdata = self.post_helper.post_request(req.toxml(encoding='utf-8'),extra_path='soap',encoding='utf-8')
-		self.assertEqual(status, 200)
+		status,reason,resdata = self.post_helper.post_request(toxml(req, 'utf-8'),extra_path='soap',encoding='utf-8')
+		self.assertEqual(status, 200, resdata)
 		res = md.parseString(resdata)
 		result_string = res.getElementsByTagName('result')[0].childNodes[0].data
 		self.assertEqual(result_string, expected_result)
-		
+
 		# latin-1 encoded request to bytesEncodingTest
-		status,reason,resdata = self.post_helper.post_request(req.toxml(encoding='latin-1'),extra_path='soap',encoding='latin-1')
-		self.assertEqual(status, 200)
+		status,reason,resdata = self.post_helper.post_request(toxml(req, 'latin-1'),extra_path='soap',encoding='latin-1')
+		self.assertEqual(status, 200, resdata)
 		res = md.parseString(resdata)
 		result_string = res.getElementsByTagName('result')[0].childNodes[0].data
 		self.assertEqual(result_string, expected_result)
 
 	def test_bytes_in_bytes_out_json(self):
-		self.string_integrety_tests_json('bytes_in_bytes_out')
-		
+		self.string_integrity_tests_json('bytes_in_bytes_out')
+
 	def test_bytes_in_uni_out_json(self):
-		self.string_integrety_tests_json('bytes_in_uni_out')
-		
+		self.string_integrity_tests_json('bytes_in_uni_out')
+
 	def test_uni_in_bytes_out_json(self):
-		self.string_integrety_tests_json('uni_in_bytes_out')
-		
+		self.string_integrity_tests_json('uni_in_bytes_out')
+
 	def test_uni_in_uni_out_json(self):
-		self.string_integrety_tests_json('uni_in_uni_out')
+		self.string_integrity_tests_json('uni_in_uni_out')
 
 	def test_bytes_in_bytes_out_soap(self):
-		self.string_integrety_tests_soap('bytes_in_bytes_out')
-		
+		self.string_integrity_tests_soap('bytes_in_bytes_out')
+
 	def test_bytes_in_uni_out_soap(self):
-		self.string_integrety_tests_soap('bytes_in_uni_out')
-		
+		self.string_integrity_tests_soap('bytes_in_uni_out')
+
 	def test_uni_in_bytes_out_soap(self):
-		self.string_integrety_tests_soap('uni_in_bytes_out')
-		
+		self.string_integrity_tests_soap('uni_in_bytes_out')
+
 	def test_uni_in_uni_out_soap(self):
-		self.string_integrety_tests_soap('uni_in_uni_out')
-
-
+		self.string_integrity_tests_soap('uni_in_uni_out')
+
+	def test_whitespace_soap(self):
+		def mangler(text):
+			return text.replace(' ', '  \n  ').replace('><', '>\n<')
+		self.string_integrity_tests_soap('uni_in_uni_out', mangler)
+
+	def test_generators(self):
+		argdict = {'methodname': 'gen_dicts', 'args': {'in_bytes': '0123456789ABCDEFG', 'count': 999}}
+		post_helper = HTTPRequestPoster('http://localhost:2376/TypeTestService')
+		status,reason,resdata = post_helper.post_request(json.dumps(argdict),encoding='utf-8')
+		self.assertEqual(status, 200, resdata)
+		LOG.debug('resdata: %r', resdata)
+
+
+def is_port_open(port, host='', timeout=1):
+	import socket
+	sock = socket.socket()
+	sock.settimeout(timeout)
+	try:
+		sock.connect((host, port))
+		sock.close()
+	except:
+		return False
+	else:
+		return True
 
 if __name__ == '__main__':
-	import servicerunner
-	servicerunner
-	service_thread = servicerunner.serve_test_service(as_thread=True)
-	unittest.main(exit=False)
-	service_thread.server.shutdown()
+	try:
+		import attachmenttests
+	except ImportError:
+		import sys, os
+		sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__),
+			'data')))
+	try:
+		from argparse import ArgumentParser
+		op = ArgumentParser()
+		op.add_argument('-v', '--verbose', action='store_true', default=False)
+		op.add_argument('--profile', action='store_true', default=False)
+		opts, args = op.parse_known_args()
+	except ImportError:
+		from optparse import OptionParser
+		op = OptionParser()
+		op.add_option('-v', '--verbose', action='store_true', default=False)
+		op.add_option('--profile', action='store_true', default=False)
+		opts, args = op.parse_args()
+
+	sys.argv[1:] = args
+	logging.basicConfig(level=(logging.DEBUG if opts.verbose else logging.INFO))
+	if is_port_open(2376):  # already running service
+		service_thread = None
+		LOG.warn('using already running service!')
+	else:
+		import servicerunner
+		service_thread = servicerunner.serve_test_service(as_thread=True)
+	try:
+		if opts.profile:
+			prof_fn = 'testladon.prof'
+			import cProfile
+			cProfile.run('unittest.main(exit=False)', prof_fn)
+			import pstats
+			p = pstats.Stats(prof_fn)
+			p.sort_stats('cumulative').print_stats(100)
+		else:
+			unittest.main(exit=False)
+	finally:
+		if service_thread:
+			service_thread.server.shutdown()