Merge lp:~pitti/apt/compressed-indexes into lp:~mvo/apt/debian-sid

[Martin Pitt]

This fixes the "apt-get update/install cache building progress goes beyond 100%" bug.

[lp:~pitti/apt/compressed-indexes]

2033. By Michael Vogt on 2010-10-13

tests/integration/test-*: remove a bunch of "local" that are used outside funtions (bash complains)

2034. By Michael Vogt on 2010-10-13

test/integration/framework: set proper dir::state::status

2035. By Michael Vogt on 2010-10-13

* apt-pkg/deb/debindexfile.cc:
  - Use FileFd::Size() instead of stat()ing the sources/binary/translations
    indexes directly, so that we have transparent handling of gzipped
    indexes.
* apt-pkg/contrib/fileutl.cc:
  - Fix FileFd::Size() for gzipped files to give the size of the
    uncompressed data. This fixes cache building progress going way
    over 100%.

=== modified file 'apt-pkg/contrib/fileutl.cc'
--- apt-pkg/contrib/fileutl.cc	2010-09-06 12:10:26 +0000
+++ apt-pkg/contrib/fileutl.cc	2010-10-13 12:35:53 +0000
@@ -915,8 +915,27 @@
 /* */
 unsigned long FileFd::Size()
 {
-   //TODO: For gz, do we need the actual file size here or the uncompressed length?
    struct stat Buf;
+   long size;
+   off_t orig_pos;
+
+   if (gz)
+   {
+       /* unfortunately zlib.h doesn't provide a gzsize(), so we have to do
+	* this ourselves; the original (uncompressed) file size is the last 32
+	* bits of the file */
+       orig_pos = lseek(iFd, 0, SEEK_CUR);
+       if (lseek(iFd, -4, SEEK_END) < 0)
+	   return _error->Errno("lseek","Unable to seek to end of gzipped file");
+       if (read(iFd, &size, 4) != 4)
+	   return _error->Errno("read","Unable to read original size of gzipped file");
+       size &= 0xFFFFFFFF;
+
+       if (lseek(iFd, orig_pos, SEEK_SET) < 0)
+	   return _error->Errno("lseek","Unable to seek in gzipped file");
+       return size;
+   }
+
    if (fstat(iFd,&Buf) != 0)
       return _error->Errno("fstat","Unable to determine the file size");
    return Buf.st_size;

=== modified file 'apt-pkg/deb/debindexfile.cc'
--- apt-pkg/deb/debindexfile.cc	2010-07-11 16:50:41 +0000
+++ apt-pkg/deb/debindexfile.cc	2010-10-13 12:35:53 +0000
@@ -149,10 +149,11 @@
 /* */
 unsigned long debSourcesIndex::Size() const
 {
-   struct stat S;
-   if (stat(IndexFile("Sources").c_str(),&S) != 0)
+   FileFd f = FileFd (IndexFile("Sources"), FileFd::ReadOnlyGzip);
+
+   if (f.Failed())
       return 0;
-   return S.st_size;
+   return f.Size();
 }
 									/*}}}*/
 
@@ -268,10 +269,11 @@
 /* This is really only used for progress reporting. */
 unsigned long debPackagesIndex::Size() const
 {
-   struct stat S;
-   if (stat(IndexFile("Packages").c_str(),&S) != 0)
+   FileFd f = FileFd (IndexFile("Packages"), FileFd::ReadOnlyGzip);
+
+   if (f.Failed())
       return 0;
-   return S.st_size;
+   return f.Size();
 }
 									/*}}}*/
 // PackagesIndex::Merge - Load the index file into a cache		/*{{{*/
@@ -458,10 +460,12 @@
 /* This is really only used for progress reporting. */
 unsigned long debTranslationsIndex::Size() const
 {
-   struct stat S;
-   if (stat(IndexFile(Language).c_str(),&S) != 0)
+   FileFd f = FileFd (IndexFile(Language), FileFd::ReadOnlyGzip);
+
+   if (f.Failed())
       return 0;
-   return S.st_size;
+
+   return f.Size();
 }
 									/*}}}*/
 // TranslationsIndex::Merge - Load the index file into a cache		/*{{{*/

=== modified file 'apt-pkg/deb/debsystem.cc'
--- apt-pkg/deb/debsystem.cc	2010-08-18 22:31:55 +0000
+++ apt-pkg/deb/debsystem.cc	2010-10-13 12:35:53 +0000
@@ -164,8 +164,8 @@
    /* These really should be jammed into a generic 'Local Database' engine
       which is yet to be determined. The functions in pkgcachegen should
       be the only users of these */
-   Cnf.CndSet("Dir::State::extended_states", Cnf.FindDir("Dir::State").append("extended_states"));
-   Cnf.CndSet("Dir::State::status", Cnf.FindDir("Dir", "/").append("var/lib/dpkg/status"));
+   Cnf.CndSet("Dir::State::extended_states", "extended_states");
+   Cnf.CndSet("Dir::State::status","/var/lib/dpkg/status");
    Cnf.CndSet("Dir::Bin::dpkg","/usr/bin/dpkg");
 
    if (StatusFile) {

=== modified file 'debian/apt.cron.daily'
--- debian/apt.cron.daily	2010-09-15 11:24:51 +0000
+++ debian/apt.cron.daily	2010-10-13 12:35:53 +0000
@@ -417,6 +417,13 @@
 # mirrors at the same time
 random_sleep
 
+# include default system language so that "apt-get update" will
+# fetch the right translated package descriptions
+if [ -r /etc/default/locale ]; then
+    . /etc/default/locale
+    export LANG LANGUAGE LC_MESSAGES LC_ALL
+fi
+
 # update package lists
 UPDATED=0
 UPDATE_STAMP=/var/lib/apt/periodic/update-stamp

=== modified file 'debian/changelog'
--- debian/changelog	2010-10-06 20:35:27 +0000
+++ debian/changelog	2010-10-13 12:35:53 +0000
@@ -5,6 +5,26 @@
   * Another typo fixed in French ("Anfin"). Thanks to bubulle
   * Wrong translation for "showauto" fixed. Thanks to Raphaël Hertzog
     Closes: #599265
+  
+  [ Michael Vogt ]
+  * debian/apt.cron.daily:
+    - source /etc/default/locale (if available) so that the 
+      apt-get update cron job fetches the right translated package
+      descriptions
+  * fix test failure on amd64
+  * apt-pkg/deb/debsystem.cc:
+    - fix issues with dir::state::status and dir::state::extended_states
+      when alternative rootdirs are used
+
+  [ Martin Pitt ]
+  * apt-pkg/deb/debindexfile.cc:
+    - Use FileFd::Size() instead of stat()ing the sources/binary/translations
+      indexes directly, so that we have transparent handling of gzipped
+      indexes.
+  * apt-pkg/contrib/fileutl.cc:
+    - Fix FileFd::Size() for gzipped files to give the size of the
+      uncompressed data. This fixes cache building progress going way
+      over 100%.
 
  -- Christian Perrier <[e-mail]>  Tue, 05 Oct 2010 05:35:58 +0200
 

=== modified file 'doc/examples/configure-index'
--- doc/examples/configure-index	2010-07-09 15:00:28 +0000
+++ doc/examples/configure-index	2010-10-13 12:35:53 +0000
@@ -433,6 +433,7 @@
   Acquire::Http "false";   // Show http command traffic
   Acquire::Https "false";   // Show https debug
   Acquire::gpgv "false";   // Show the gpgv traffic
+  Acquire::cdrom "false";   // Show cdrom debug output
   aptcdrom "false";        // Show found package files
   IdentCdrom "false";
   acquire::netrc "false";  // netrc parser

=== modified file 'test/integration/framework'
--- test/integration/framework	2010-09-29 17:55:04 +0000
+++ test/integration/framework	2010-10-13 12:35:53 +0000
@@ -64,6 +64,8 @@
 	msgdebug "Executing: ${CCMD}$*${CDEBUG} "
 	if [ -f ./aptconfig.conf ]; then
 		APT_CONFIG=aptconfig.conf LD_LIBRARY_PATH=${BUILDDIRECTORY} ${BUILDDIRECTORY}/$*
+        elif [ -f ../aptconfig.conf ]; then
+                APT_CONFIG=../aptconfig.conf LD_LIBRARY_PATH=${BUILDDIRECTORY} ${BUILDDIRECTORY}/$*
 	else
 		LD_LIBRARY_PATH=${BUILDDIRECTORY} ${BUILDDIRECTORY}/$*
 	fi