Merge lp:~psusi/ubuntu/precise/ureadahead/faster into lp:ubuntu/precise/ureadahead

[Phillip Susi]

These are some changes I developed a year ago to make ureadahead even faster. I forgot to propose merging them in oneiric, so I've rebased them for precise.

[Luke Yelavich (themuso)]

This FTBF in precise. As per IRC, this will be looked into.

review: Needs Fixing

[Evan Broder (broder)]

Changing the status of this to "Work in progress" to remove it from the sponsor queue, as it still FTBFS. Please feel free to change the status back to "Needs review" when it's ready to be reviewed.

Unmerged revisions

29. By Phillip Susi on 2011-12-06

Force reprofile on upgrade

28. By Phillip Susi on 2011-12-06

* Read all directories optimally
* trace_add_path(): accept directories
* Readahead on the raw block device
* Made preload_inode_group() smarter

27. By Phillip Susi on 2010-12-24

Read all directories optimally

When tracing, add each directory in the path to a file when
adding it to the pack. When booting, read the directories
via the block device, and normal files the old way, in two
phases. This is required because ext3/4 cache directories
via the block device mapping and normal files via file
mappings. Reading the directories into the cache speeds up
the open() calls to the normal files. To facillitate this,
the st_mode has been added to the file entry in the pack
file so ureadahead can see if the file is a regular file
or a directory.

26. By Phillip Susi on 2010-12-24

trace_add_path(): accept directories

Rather than bail out and ignore a directory, it now
directly adds the extents for the entire directory
to the pack file.

25. By Phillip Susi on 2010-12-24

Readahead on the raw block device

As long as we managed to open the raw block device and preload
the inode tables, don't bother open()ing the files and instead
call readahead() on the raw block device with the physical
offsets instead. This makes sure we don't block in the open()
calls.

24. By Phillip Susi on 2010-12-24

Made preload_inode_group() smarter

Before it called on e2fslibs to get the next inode until it returned
one not in the specified group. This caused sync reads of the inodes
and could read inode tables that we don't need. Now it looks up the
locations of the inode table, inode bitmap, and block bitmap and calls
readahead() on the block device for those locations to queue their
reads in the background.

=== modified file 'debian/changelog'
--- debian/changelog	2011-03-16 17:19:01 +0000
+++ debian/changelog	2011-12-06 03:33:24 +0000
@@ -1,3 +1,13 @@
+ureadahead (0.100.0-12) precise; urgency=low
+
+  * Read all directories optimally
+  * trace_add_path(): accept directories
+  * Readahead on the raw block device
+  * Made preload_inode_group() smarter
+  * Force reprofile on upgrade
+
+ -- Phillip Susi <[e-mail]>  Thu, 23 Dec 2010 19:48:59 -0500
+
 ureadahead (0.100.0-11) natty; urgency=low
 
   * src/trace.c: leave room for string termination on reads (LP: #485194).
 
=== modified file 'debian/ureadahead.postinst'
--- debian/ureadahead.postinst	2009-11-09 18:38:51 +0000
+++ debian/ureadahead.postinst	2011-12-06 03:33:24 +0000
@@ -20,10 +20,7 @@
 
 
 case "$1" in
-    configure)
-	;;
-
-    triggered)
+    configure|triggered)
 	# Force a reprofile
 	if [ -f /var/lib/ureadahead/pack ]; then
 	    echo "ureadahead will be reprofiled on next reboot"
 
=== modified file 'src/pack.c'
--- src/pack.c	2010-06-25 12:00:39 +0000
+++ src/pack.c	2011-12-06 03:33:24 +0000
@@ -88,7 +88,7 @@
 /* Prototypes for static functions */
 static void  print_time          (const char *message, struct timespec *start);
 static int   do_readahead_hdd    (PackFile *file, int daemonise);
-static void  preload_inode_group (ext2_filsys fs, int group);
+static void  preload_inode_group (ext2_filsys fs, int fd, int group);
 static int   do_readahead_ssd    (PackFile *file, int daemonise);
 static void *ra_thread           (void *ptr);
 
@@ -689,6 +689,7 @@
 	const char *    devname;
 	ext2_filsys     fs = NULL;
 	nih_local int * fds = NULL;
+	int blockdev;
 
 	nih_assert (file != NULL);
 
@@ -713,26 +714,33 @@
 	 */
 	devname = blkid_devno_to_devname (file->dev);
 	if (devname
-	    && (! ext2fs_open (devname, 0, 0, 0, unix_io_manager, &fs))) {
+	    && (! ext2fs_open (devname, 0, 0, 0, unix_io_manager, &fs))
+	    && ((blockdev = open (devname, O_RDONLY )) > 0)) {
 		nih_assert (fs != NULL);
 
 		for (size_t i = 0; i < file->num_groups; i++)
-			preload_inode_group (fs, file->groups[i]);
+		  preload_inode_group (fs, blockdev, file->groups[i]);
 
 		ext2fs_close (fs);
 	}
 
 	print_time ("Preload ext2fs inodes", &start);
-
+	if (blockdev > 0)
+		for (size_t i = 0; i < file->num_blocks; i++)
+			if (S_ISDIR(file->paths[file->blocks[i].pathidx].st_mode))
+				readahead (blockdev,
+					   file->blocks[i].physical,
+					   file->blocks[i].length);
 	/* Open all of the files */
 	fds = NIH_MUST (nih_alloc (NULL, sizeof (int) * file->num_paths));
 	for (size_t i = 0; i < file->num_paths; i++) {
+		if (S_ISDIR(file->paths[i].st_mode))
+			continue;
 		fds[i] = open (file->paths[i].path, O_RDONLY | O_NOATIME);
 		if (fds[i] < 0)
 			nih_warn ("%s: %s", file->paths[i].path,
 				  strerror (errno));
 	}
-
 	print_time ("Open files", &start);
 
 	/* Read in all of the blocks in a single pass for rotational
@@ -741,9 +749,9 @@
 	 */
 	for (size_t i = 0; i < file->num_blocks; i++) {
 		if ((fds[file->blocks[i].pathidx] < 0)
+		    || S_ISDIR(file->paths[file->blocks[i].pathidx].st_mode)
 		    || (file->blocks[i].pathidx >= file->num_paths))
 			continue;
-
 		readahead (fds[file->blocks[i].pathidx],
 			   file->blocks[i].offset,
 			   file->blocks[i].length);
@@ -756,26 +764,27 @@
 
 static void
 preload_inode_group (ext2_filsys fs,
-		     int         group)
+		     int fd,
+		     int group)
 {
 	ext2_inode_scan scan = NULL;
 
 	nih_assert (fs != NULL);
-
-	if (! ext2fs_open_inode_scan (fs, 0, &scan)) {
-		nih_assert (scan != NULL);
-
-		if (! ext2fs_inode_scan_goto_blockgroup (scan, group)) {
-			struct ext2_inode inode;
-			ext2_ino_t        ino = 0;
-
-			while ((! ext2fs_get_next_inode (scan, &ino, &inode))
-			       && (ext2fs_group_of_ino (fs, ino) == group))
-				;
-		}
-
-		ext2fs_close_inode_scan (scan);
-	}
+	int itable_size;
+	if (EXT2_HAS_RO_COMPAT_FEATURE (fs->super, EXT4_FEATURE_RO_COMPAT_GDT_CSUM)) {
+	  itable_size = ((EXT2_INODES_PER_GROUP(fs->super)
+			  -fs->group_desc[group].bg_itable_unused)
+			 * EXT2_INODE_SIZE(fs->super));
+	} else itable_size = EXT2_INODES_PER_GROUP(fs->super) * EXT2_INODE_SIZE(fs->super);
+	readahead (fd,
+		   fs->group_desc[group].bg_block_bitmap * EXT2_BLOCK_SIZE(fs->super),
+		   (EXT2_BLOCKS_PER_GROUP(fs->super)+7) >> 3);
+	readahead (fd,
+		   fs->group_desc[group].bg_inode_bitmap * EXT2_BLOCK_SIZE(fs->super),
+		   (EXT2_INODES_PER_GROUP(fs->super)+7) >> 3);
+	readahead (fd,
+		   fs->group_desc[group].bg_inode_table * EXT2_BLOCK_SIZE(fs->super),
+		   itable_size);
 }
 
 
 
=== modified file 'src/pack.h'
--- src/pack.h	2009-11-09 18:38:51 +0000
+++ src/pack.h	2011-12-06 03:33:24 +0000
@@ -42,6 +42,7 @@
 typedef struct pack_path {
 	int   group;
 	ino_t ino;
+	mode_t st_mode;
 	char  path[PACK_PATH_MAX+1];
 } PackPath;
 
 
=== modified file 'src/trace.c'
--- src/trace.c	2010-09-20 17:27:30 +0000
+++ src/trace.c	2011-12-06 03:33:24 +0000
@@ -374,7 +374,12 @@
 
 		fix_path (ptr);
 		trace_add_path (parent, ptr, files, num_files);
-
+		for (int i = strlen (ptr) - 1; i; i--) {
+			if (ptr[i] == '/') {
+				ptr[i] = 0;
+				trace_add_path (parent, ptr, files, num_files);
+			}
+		}
 		nih_free (line);
 	}
 
@@ -496,7 +501,8 @@
 	if ((lstat (pathname, &statbuf) < 0)
 	    || (S_ISLNK (statbuf.st_mode)
 		&& (stat (pathname, &statbuf) < 0))
-	    || (! S_ISREG (statbuf.st_mode)))
+	    || (!( S_ISREG (statbuf.st_mode)
+		   || S_ISDIR (statbuf.st_mode))))
 		return 0;
 
 	/* Open and stat again to get the genuine details, in case it
@@ -519,7 +525,8 @@
 	}
 
 	/* Double-check that it's really still a file */
-	if (! S_ISREG (statbuf.st_mode)) {
+	if (! (S_ISREG (statbuf.st_mode)
+	       || S_ISDIR (statbuf.st_mode))) {
 		close (fd);
 		return 0;
 	}
@@ -545,6 +552,7 @@
 
 	path->group = -1;
 	path->ino = statbuf.st_ino;
+	path->st_mode = statbuf.st_mode;
 
 	strncpy (path->path, pathname, PACK_PATH_MAX);
 	path->path[PACK_PATH_MAX] = '\0';
@@ -585,9 +593,14 @@
 	}
 
 	/* Now read the in-memory chunks of this file and add those to
-	 * the pack file too.
+	 * the pack file too.  For directories, just add the whole
+	 * thing for rotational disks.
 	 */
-	trace_add_chunks (*files, file, path, fd, statbuf.st_size);
+	if (S_ISREG (statbuf.st_mode))
+		trace_add_chunks (*files, file, path, fd, statbuf.st_size);
+	else if (file->rotational)
+		trace_add_extents (*files, file, path, fd, statbuf.st_size,
+				   0, statbuf.st_size);
 	close (fd);
 
 	return 0;