Merge lp:~psusi/ubuntu/precise/ureadahead/faster into lp:ubuntu/precise/ureadahead

Proposed by Phillip Susi
Status: Work in progress
Proposed branch: lp:~psusi/ubuntu/precise/ureadahead/faster
Merge into: lp:ubuntu/precise/ureadahead
Diff against target: 222 lines (+61/-31)
5 files modified
debian/changelog (+10/-0)
debian/ureadahead.postinst (+1/-4)
src/pack.c (+31/-22)
src/pack.h (+1/-0)
src/trace.c (+18/-5)
To merge this branch: bzr merge lp:~psusi/ubuntu/precise/ureadahead/faster
Reviewer Review Type Date Requested Status
Luke Yelavich (community) Needs Fixing
James Hunt Pending
Ubuntu branches Pending
Review via email: mp+84556@code.launchpad.net

Description of the change

These are some changes I developed a year ago to make ureadahead even faster. I forgot to propose merging them in oneiric, so I've rebased them for precise.

To post a comment you must log in.
Revision history for this message
Luke Yelavich (themuso) wrote :

This FTBF in precise. As per IRC, this will be looked into.

review: Needs Fixing
Revision history for this message
Evan Broder (broder) wrote :

Changing the status of this to "Work in progress" to remove it from the sponsor queue, as it still FTBFS. Please feel free to change the status back to "Needs review" when it's ready to be reviewed.

Unmerged revisions

29. By Phillip Susi

Force reprofile on upgrade

28. By Phillip Susi

* Read all directories optimally
* trace_add_path(): accept directories
* Readahead on the raw block device
* Made preload_inode_group() smarter

27. By Phillip Susi

Read all directories optimally

When tracing, add each directory in the path to a file when
adding it to the pack. When booting, read the directories
via the block device, and normal files the old way, in two
phases. This is required because ext3/4 cache directories
via the block device mapping and normal files via file
mappings. Reading the directories into the cache speeds up
the open() calls to the normal files. To facillitate this,
the st_mode has been added to the file entry in the pack
file so ureadahead can see if the file is a regular file
or a directory.

26. By Phillip Susi

trace_add_path(): accept directories

Rather than bail out and ignore a directory, it now
directly adds the extents for the entire directory
to the pack file.

25. By Phillip Susi

Readahead on the raw block device

As long as we managed to open the raw block device and preload
the inode tables, don't bother open()ing the files and instead
call readahead() on the raw block device with the physical
offsets instead. This makes sure we don't block in the open()
calls.

24. By Phillip Susi

Made preload_inode_group() smarter

Before it called on e2fslibs to get the next inode until it returned
one not in the specified group. This caused sync reads of the inodes
and could read inode tables that we don't need. Now it looks up the
locations of the inode table, inode bitmap, and block bitmap and calls
readahead() on the block device for those locations to queue their
reads in the background.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'debian/changelog'
2--- debian/changelog 2011-03-16 17:19:01 +0000
3+++ debian/changelog 2011-12-06 03:33:24 +0000
4@@ -1,3 +1,13 @@
5+ureadahead (0.100.0-12) precise; urgency=low
6+
7+ * Read all directories optimally
8+ * trace_add_path(): accept directories
9+ * Readahead on the raw block device
10+ * Made preload_inode_group() smarter
11+ * Force reprofile on upgrade
12+
13+ -- Phillip Susi <psusi@cfl.rr.com> Thu, 23 Dec 2010 19:48:59 -0500
14+
15 ureadahead (0.100.0-11) natty; urgency=low
16
17 * src/trace.c: leave room for string termination on reads (LP: #485194).
18
19=== modified file 'debian/ureadahead.postinst'
20--- debian/ureadahead.postinst 2009-11-09 18:38:51 +0000
21+++ debian/ureadahead.postinst 2011-12-06 03:33:24 +0000
22@@ -20,10 +20,7 @@
23
24
25 case "$1" in
26- configure)
27- ;;
28-
29- triggered)
30+ configure|triggered)
31 # Force a reprofile
32 if [ -f /var/lib/ureadahead/pack ]; then
33 echo "ureadahead will be reprofiled on next reboot"
34
35=== modified file 'src/pack.c'
36--- src/pack.c 2010-06-25 12:00:39 +0000
37+++ src/pack.c 2011-12-06 03:33:24 +0000
38@@ -88,7 +88,7 @@
39 /* Prototypes for static functions */
40 static void print_time (const char *message, struct timespec *start);
41 static int do_readahead_hdd (PackFile *file, int daemonise);
42-static void preload_inode_group (ext2_filsys fs, int group);
43+static void preload_inode_group (ext2_filsys fs, int fd, int group);
44 static int do_readahead_ssd (PackFile *file, int daemonise);
45 static void *ra_thread (void *ptr);
46
47@@ -689,6 +689,7 @@
48 const char * devname;
49 ext2_filsys fs = NULL;
50 nih_local int * fds = NULL;
51+ int blockdev;
52
53 nih_assert (file != NULL);
54
55@@ -713,26 +714,33 @@
56 */
57 devname = blkid_devno_to_devname (file->dev);
58 if (devname
59- && (! ext2fs_open (devname, 0, 0, 0, unix_io_manager, &fs))) {
60+ && (! ext2fs_open (devname, 0, 0, 0, unix_io_manager, &fs))
61+ && ((blockdev = open (devname, O_RDONLY )) > 0)) {
62 nih_assert (fs != NULL);
63
64 for (size_t i = 0; i < file->num_groups; i++)
65- preload_inode_group (fs, file->groups[i]);
66+ preload_inode_group (fs, blockdev, file->groups[i]);
67
68 ext2fs_close (fs);
69 }
70
71 print_time ("Preload ext2fs inodes", &start);
72-
73+ if (blockdev > 0)
74+ for (size_t i = 0; i < file->num_blocks; i++)
75+ if (S_ISDIR(file->paths[file->blocks[i].pathidx].st_mode))
76+ readahead (blockdev,
77+ file->blocks[i].physical,
78+ file->blocks[i].length);
79 /* Open all of the files */
80 fds = NIH_MUST (nih_alloc (NULL, sizeof (int) * file->num_paths));
81 for (size_t i = 0; i < file->num_paths; i++) {
82+ if (S_ISDIR(file->paths[i].st_mode))
83+ continue;
84 fds[i] = open (file->paths[i].path, O_RDONLY | O_NOATIME);
85 if (fds[i] < 0)
86 nih_warn ("%s: %s", file->paths[i].path,
87 strerror (errno));
88 }
89-
90 print_time ("Open files", &start);
91
92 /* Read in all of the blocks in a single pass for rotational
93@@ -741,9 +749,9 @@
94 */
95 for (size_t i = 0; i < file->num_blocks; i++) {
96 if ((fds[file->blocks[i].pathidx] < 0)
97+ || S_ISDIR(file->paths[file->blocks[i].pathidx].st_mode)
98 || (file->blocks[i].pathidx >= file->num_paths))
99 continue;
100-
101 readahead (fds[file->blocks[i].pathidx],
102 file->blocks[i].offset,
103 file->blocks[i].length);
104@@ -756,26 +764,27 @@
105
106 static void
107 preload_inode_group (ext2_filsys fs,
108- int group)
109+ int fd,
110+ int group)
111 {
112 ext2_inode_scan scan = NULL;
113
114 nih_assert (fs != NULL);
115-
116- if (! ext2fs_open_inode_scan (fs, 0, &scan)) {
117- nih_assert (scan != NULL);
118-
119- if (! ext2fs_inode_scan_goto_blockgroup (scan, group)) {
120- struct ext2_inode inode;
121- ext2_ino_t ino = 0;
122-
123- while ((! ext2fs_get_next_inode (scan, &ino, &inode))
124- && (ext2fs_group_of_ino (fs, ino) == group))
125- ;
126- }
127-
128- ext2fs_close_inode_scan (scan);
129- }
130+ int itable_size;
131+ if (EXT2_HAS_RO_COMPAT_FEATURE (fs->super, EXT4_FEATURE_RO_COMPAT_GDT_CSUM)) {
132+ itable_size = ((EXT2_INODES_PER_GROUP(fs->super)
133+ -fs->group_desc[group].bg_itable_unused)
134+ * EXT2_INODE_SIZE(fs->super));
135+ } else itable_size = EXT2_INODES_PER_GROUP(fs->super) * EXT2_INODE_SIZE(fs->super);
136+ readahead (fd,
137+ fs->group_desc[group].bg_block_bitmap * EXT2_BLOCK_SIZE(fs->super),
138+ (EXT2_BLOCKS_PER_GROUP(fs->super)+7) >> 3);
139+ readahead (fd,
140+ fs->group_desc[group].bg_inode_bitmap * EXT2_BLOCK_SIZE(fs->super),
141+ (EXT2_INODES_PER_GROUP(fs->super)+7) >> 3);
142+ readahead (fd,
143+ fs->group_desc[group].bg_inode_table * EXT2_BLOCK_SIZE(fs->super),
144+ itable_size);
145 }
146
147
148
149=== modified file 'src/pack.h'
150--- src/pack.h 2009-11-09 18:38:51 +0000
151+++ src/pack.h 2011-12-06 03:33:24 +0000
152@@ -42,6 +42,7 @@
153 typedef struct pack_path {
154 int group;
155 ino_t ino;
156+ mode_t st_mode;
157 char path[PACK_PATH_MAX+1];
158 } PackPath;
159
160
161=== modified file 'src/trace.c'
162--- src/trace.c 2010-09-20 17:27:30 +0000
163+++ src/trace.c 2011-12-06 03:33:24 +0000
164@@ -374,7 +374,12 @@
165
166 fix_path (ptr);
167 trace_add_path (parent, ptr, files, num_files);
168-
169+ for (int i = strlen (ptr) - 1; i; i--) {
170+ if (ptr[i] == '/') {
171+ ptr[i] = 0;
172+ trace_add_path (parent, ptr, files, num_files);
173+ }
174+ }
175 nih_free (line);
176 }
177
178@@ -496,7 +501,8 @@
179 if ((lstat (pathname, &statbuf) < 0)
180 || (S_ISLNK (statbuf.st_mode)
181 && (stat (pathname, &statbuf) < 0))
182- || (! S_ISREG (statbuf.st_mode)))
183+ || (!( S_ISREG (statbuf.st_mode)
184+ || S_ISDIR (statbuf.st_mode))))
185 return 0;
186
187 /* Open and stat again to get the genuine details, in case it
188@@ -519,7 +525,8 @@
189 }
190
191 /* Double-check that it's really still a file */
192- if (! S_ISREG (statbuf.st_mode)) {
193+ if (! (S_ISREG (statbuf.st_mode)
194+ || S_ISDIR (statbuf.st_mode))) {
195 close (fd);
196 return 0;
197 }
198@@ -545,6 +552,7 @@
199
200 path->group = -1;
201 path->ino = statbuf.st_ino;
202+ path->st_mode = statbuf.st_mode;
203
204 strncpy (path->path, pathname, PACK_PATH_MAX);
205 path->path[PACK_PATH_MAX] = '\0';
206@@ -585,9 +593,14 @@
207 }
208
209 /* Now read the in-memory chunks of this file and add those to
210- * the pack file too.
211+ * the pack file too. For directories, just add the whole
212+ * thing for rotational disks.
213 */
214- trace_add_chunks (*files, file, path, fd, statbuf.st_size);
215+ if (S_ISREG (statbuf.st_mode))
216+ trace_add_chunks (*files, file, path, fd, statbuf.st_size);
217+ else if (file->rotational)
218+ trace_add_extents (*files, file, path, fd, statbuf.st_size,
219+ 0, statbuf.st_size);
220 close (fd);
221
222 return 0;

Subscribers

People subscribed via source and target branches

to all changes: