Merge lp:~fmos/zim/bugfixes into lp:~jaap.karssenberg/zim/pyzim

[Fabian Stanke]

Improved verbatim handling

This deals exclusively with improved verbatim handling.
It implements all definitions of the blueprint http://www.zim-wiki.org/wiki/doku.php?id=verbatim_syntax
and fixes #297932 , #586296 and partly also #589525

[Jaap Karssenberg (jaap.karssenberg)]

Thanks for submitting this patch. I'll be mostly offline for the next
~10 days, will go over this patch in detail when I'm back.

-- Jaap

[Jaap Karssenberg (jaap.karssenberg)]

Will make some small style adjustments, but logic looks good to me. Thanks again for looking into this.

review: Approve

[sneakypete (sneakypete81)]

On 13 August 2010 16:57, Fabian Moser <email address hidden> wrote:
> Fabian Moser has proposed merging lp:~fmos/zim/bugfixes into lp:zim.
> This deals exclusively with improved verbatim handling.
> It implements all definitions of the blueprint http://www.zim-wiki.org/wiki/doku.php?id=verbatim_syntax
> and fixes #297932 , #586296 and partly also #589525

Thanks Fabian, I've been looking forward to this for a while. Zim gets
better every month!

=== modified file 'zim/formats/__init__.py'
--- zim/formats/__init__.py	2010-07-06 20:36:53 +0000
+++ zim/formats/__init__.py	2010-08-13 15:57:43 +0000
@@ -411,14 +411,14 @@
 			self._seen_eol = need_eol
 
 		# Fix prefix newlines
-		if self._tail and self._last.tag in ('h', 'p', 'pre') \
+		if self._tail and self._last.tag in ('h', 'p') \
 		and not text.startswith('\n'):
 			if text:
 				text = '\n' + text
 			else:
 				text = '\n'
 				self._seen_eol = 1
-		if self._tail and self._last.tag == 'li' \
+		elif self._tail and self._last.tag == 'li' \
 		and text.startswith('\n'):
 			text = text[1:]
 			if not text.strip('\n'):

=== modified file 'zim/formats/wiki.py'
--- zim/formats/wiki.py	2010-06-14 21:03:10 +0000
+++ zim/formats/wiki.py	2010-08-13 15:57:43 +0000
@@ -39,8 +39,8 @@
 	bullet_types[bullets[bullet]] = bullet
 
 parser_re = {
-	'blockstart': re.compile("\A(\t*''')\s*?\n"),
-	'pre':        re.compile("\A(?P<escape>\t*''')\s*?(?P<content>^.*?)^(?P=escape)\s*\Z", re.M | re.S),
+	'blockstart': re.compile("^(\t*''')\s*?\n", re.M),
+	'pre':        re.compile("^(?P<escape>\t*''')\s*?(?P<content>^.*?)^(?P=escape)\s*\n", re.M | re.S),
 	'splithead':  re.compile('^(==+[ \t]+\S.*?\n)', re.M),
 	'heading':    re.compile("\A((==+)[ \t]+(.*?)([ \t]+==+)?[ \t]*\n?)\Z"),
 	'splitlist':  re.compile("((?:^[ \t]*(?:%s)[ \t]+.*\n?)+)" % bullet_re, re.M),
@@ -107,11 +107,21 @@
 			# Else append empty paragraph to start new para
 			paras.append('')
 			return True
-
+		
+		def blocks_closed():
+			# This function checks if there are unfinished blocks in the last 
+			# paragraph.
+			if len(paras[-1]) == 0:
+				return True
+			# Eliminate closed blocks
+			nonblock = parser_re['pre'].split(paras[-1])
+			#  Blocks are closed if none is opened at the end
+			return parser_re['blockstart'].search(nonblock[-1]) == None
+			
 		para_isspace = False
 		for line in input:
 			# Try start new para when switching between text and empty lines or back
-			if line.isspace() != para_isspace or parser_re['blockstart'].match(line):
+			if line.isspace() != para_isspace and blocks_closed():
 				if para_start():
 					para_isspace = line.isspace() # decide type of new para
 			paras[-1] += line
@@ -126,19 +136,24 @@
 			# exceptions like it (crosses fingers)
 			para = para.replace(u'\u2028', '\n')
 
-			if not self.backward and parser_re['blockstart'].search(para):
-				self._parse_block(builder, para)
-			elif self.backward and not para.isspace() \
+			if self.backward and not para.isspace() \
 			and not parser_re['unindented_line'].search(para):
 				self._parse_block(builder, para)
 			else:
-				parts = parser_re['splithead'].split(para)
-				for i, p in enumerate(parts):
-					if i % 2:
-						# odd elements in the list are headings after split
-						self._parse_head(builder, p)
-					elif len(p) > 0:
-						self._parse_para(builder, p)
+				block_parts = parser_re['pre'].split(para)
+				for i, b in enumerate(block_parts):
+					if i % 3 == 0:
+						# Text paragraph
+						parts = parser_re['splithead'].split(b)
+						for j, p in enumerate(parts):
+							if j % 2:
+								# odd elements in the list are headings after split
+								self._parse_head(builder, p)
+							elif len(p) > 0:
+								self._parse_para(builder, p)
+					elif i % 3 == 1:
+						# Block
+						self._parse_block(builder, b + '\n' + block_parts[i+1] + b + '\n')
 
 		builder.end('zim-tree')
 		return ParseTree(builder.close())

=== modified file 'zim/gui/pageview.py'
--- zim/gui/pageview.py	2010-08-06 21:10:48 +0000
+++ zim/gui/pageview.py	2010-08-13 15:57:43 +0000
@@ -1079,7 +1079,9 @@
 		# First call parent for the actual insert
 		if string == '\n':
 			# Break tags that are not allowed to span over multiple lines
-			self._editmode_tags = filter(_is_not_style_tag, self._editmode_tags)
+			self._editmode_tags = filter(
+				lambda tag: _is_pre_tag(tag) or _is_not_style_tag(tag), 
+				self._editmode_tags)
 			self._editmode_tags = filter(_is_not_link_tag, self._editmode_tags)
 			self.emit('textstyle-changed', None)
 			# TODO make this more robust for multiline inserts
@@ -2194,7 +2196,8 @@
 			# Tab at start of line indents
 			iter = buffer.get_insert_iter()
 			home, ourhome = self.get_visual_home_positions(iter)
-			if home.starts_line() and iter.compare(ourhome) < 1:
+			if home.starts_line() and iter.compare(ourhome) < 1 \
+			and not filter(_is_pre_tag, iter.get_tags()):
 				row, list = TextBufferList.new_from_iter(buffer, iter)
 				if list and self.preferences['recursive_indentlist']:
 					list.indent(row)
@@ -2208,7 +2211,8 @@
 			# Backspace or Ctrl-Tab unindents line
 			iter = buffer.get_iter_at_mark(buffer.get_insert())
 			home, ourhome = self.get_visual_home_positions(iter)
-			if home.starts_line() and iter.compare(ourhome) < 1:
+			if home.starts_line() and iter.compare(ourhome) < 1 \
+			and not filter(_is_pre_tag, iter.get_tags()):
 				row, list = TextBufferList.new_from_iter(buffer, iter)
 				if list and self.preferences['recursive_indentlist']:
 					done = list.unindent(row)
@@ -2304,21 +2308,53 @@
 		#   > Quotes whole selection with '>'
 		handled = True
 		buffer = self.get_buffer()
-
+			
+		def delete_char(iter):
+			# Deletes the character at the iterator position
+			next = iter.copy()
+			if next.forward_char():
+				buffer.delete(iter, next)
+			
 		def decrement_indent():
-			# For selection decrement first check if all lines have indent
-			level = []
-			buffer.strip_selection()
-			buffer.foreach_line_in_selection(
-				lambda i: level.append(buffer.get_indent(i)) )
-			if level and min(level) > 0:
-				return buffer.foreach_line_in_selection(buffer.decrement_indent)
+			# Check if inside verbatim block AND entire selection without tag toggle
+			iter = buffer.get_insert_iter()
+			if filter(_is_pre_tag, iter.get_tags()) \
+			and not find_tag_toggle():
+				missing_tabs = []
+				check_tab = lambda iter: (iter.get_char() == '\t') or missing_tabs.append(1) 
+				buffer.foreach_line_in_selection(check_tab)
+				if len(missing_tabs) == 0:
+					return buffer.foreach_line_in_selection(delete_char)
+				else:
+					return False
 			else:
-				return False
+				# For selection decrement first check if all lines have indent
+				level = []
+				buffer.strip_selection()
+				buffer.foreach_line_in_selection(
+					lambda i: level.append(buffer.get_indent(i)) )
+				if level and min(level) > 0:
+					return buffer.foreach_line_in_selection(buffer.decrement_indent)
+				else:
+					return False
+			
+		def find_tag_toggle():
+			# Checks if there are any tag changes within the selection 
+			start, end = buffer.get_selection_bounds()
+			toggle = start.copy()
+			toggle.forward_to_tag_toggle(None)
+			return toggle.compare(end) < 0
 
 		with buffer.user_action:
 			if event.keyval in KEYVALS_TAB:
-				buffer.foreach_line_in_selection(buffer.increment_indent)
+				# Check if inside verbatim block AND entire selection without tag toggle
+				iter = buffer.get_insert_iter()
+				if filter(_is_pre_tag, iter.get_tags()) \
+				and not find_tag_toggle():
+					prepend_tab = lambda iter: buffer.insert(iter, '\t')
+					buffer.foreach_line_in_selection(prepend_tab)
+				else:
+					buffer.foreach_line_in_selection(buffer.increment_indent)
 			elif event.keyval in KEYVALS_LEFT_TAB:
 				decrement_indent()
 			elif event.keyval in KEYVALS_BACKSPACE \