diff -Nru duplicity-0.8.22/bin/duplicity.1 duplicity-0.8.23/bin/duplicity.1 --- duplicity-0.8.22/bin/duplicity.1 2022-03-04 18:52:39.000000000 +0000 +++ duplicity-0.8.23/bin/duplicity.1 2022-05-15 15:34:02.000000000 +0000 @@ -150,7 +150,7 @@ If a backup is in order and old signatures can be found duplicity automatically performs an incremental backup. .PP -.B Note: +.B NOTE: The following explanations explain some but .B not all options that can be used in connection with that action command. @@ -282,7 +282,9 @@ .TP .BI "--archive-dir " path +.RS The archive directory. + .B NOTE: This option changed in 0.6.0. The archive directory is now necessary in order to manage persistence for current and future enhancements. @@ -329,6 +331,7 @@ --archive-dir=/arch, --name=foo /arch/foo .RE +.RE .TP .BI "--asynchronous-upload " @@ -342,11 +345,33 @@ enough storage space is required to store two volumes. .TP +.BI "--azure-blob-tier" +Standard storage tier used for backup files (Hot|Cool|Archive). + +.TP +.BI "--azure-max-single-put-size" +Specify the number of the largest supported upload size where the Azure +library makes only one put call. If the content size is known and below this +value the Azure library will only perform one put request to upload one block. +The number is expected to be in bytes. + +.TP +.BI "--azure-max-block-size" +Specify the number for the block size used by the Azure library to upload +blobs if it is split into multiple blocks. +The maximum block size the service supports is 104857600 (100MiB) and the +default is 4194304 (4MiB) + +.TP +.BI "--azure-max-connections" +Specify the number of maximum connections to transfer one blob to Azure +blob size exceeds 64MB. The default values is 2. + +.TP .BI "--backend-retry-delay " number Specifies the number of seconds that duplicity waits after an error has occured before attempting to repeat the operation. - .TP .BI "--cf-backend " backend Allows the explicit selection of a cloudfiles backend. Defaults to @@ -388,7 +413,6 @@ .BR gpg (1), section "HOW TO SPECIFY A USER ID" for details. - .TP .BI "--encrypt-secret-keyring " filename This option can only be used with @@ -463,8 +487,18 @@ section for more information. .TP -.BI "--file-prefix, --file-prefix-manifest, --file-prefix-archive, --file-prefix-signature -Adds a prefix to all files, manifest files, archive files, and/or signature files. +.BI --file-prefix " prefix" +.PD 0 +.TP +.BI --file-prefix-manifest " prefix" +.PD 0 +.TP +.BI --file-prefix-archive " prefix" +.PD 0 +.TP +.BI --file-prefix-signature " prefix" +.RS +Adds a prefix to either all files or only manifest, archive, signature files. The same set of prefixes must be passed in on backup and restore. @@ -473,6 +507,7 @@ See also .B "A NOTE ON FILENAME PREFIXES" +.RE .TP .BI "--file-to-restore " path @@ -520,9 +555,9 @@ automatically try all available secret keys in order to decrypt the backup. See gpg(1) for more details. - .TP .BI --ignore-errors +.RS Try to ignore certain errors if they happen. This option is only intended to allow the restoration of a backup in the face of certain problems that would otherwise cause the backup to fail. It is not ever @@ -541,6 +576,7 @@ understood or understood but not your own responsibility, please contact duplicity maintainers. The need to use this option under production circumstances would normally be considered a bug. +.RE .TP .BI "--imap-full-address " email_address @@ -609,6 +645,7 @@ .TP .BI "--max-blocksize " number +.RS determines the number of the blocks examined for changes during the diff process. For files < 1MB the blocksize is a constant of 512. For files over 1MB the size is given by: @@ -621,9 +658,11 @@ If you specify a larger max_blocksize, your difftar files will be larger, but your sigtar files will be smaller. If you specify a smaller max_blocksize, the reverse occurs. The --max-blocksize option should be in multiples of 512. +.RE .TP .BI "--name " symbolicname +.RS Set the symbolic name of the backup being operated on. The intent is to use a separate name for each logically distinct backup. For example, someone may use "home_daily_s3" for the daily backup of a @@ -635,6 +674,7 @@ more than one distinct backup are encouraged to use this option. If not specified, the default value is a hash of the backend URL. +.RE .TP .BI --no-compression @@ -717,6 +757,7 @@ .TP .BI "--rename " " " +.RS Treats the path .I orig in the backup as if it were the path @@ -724,9 +765,11 @@ Can be passed multiple times. An example: duplicity restore --rename Documents/metal Music/metal sftp://uid@other.host/some_dir /home/me +.RE .TP .BI "--rsync-options " options +.RS Allows you to pass options to the rsync backend. The .I options list should be of the form "opt1=parm1 opt2=parm2" where the option string is @@ -735,81 +778,148 @@ designating the remote port to use. Here is a possibly useful example: duplicity --rsync-options="--partial-dir=.rsync-partial" /home/me rsync://uid@other.host/some_dir +.RE + +.TP +.BI "--s3-endpoint-url" " url" +.RS +Specifies the endpoint URL of the S3 storage. + +.B NOTE: +Due to API restrictions the legacy backend +.B boto +will use only the values +.I scheme (protocol) +and +.I hostname +from the given +.IR url . +Choosing +.B 'http://' +will disable SSL encryption, just as if +.B --s3-unencrypted-connection +were set. +.RE .TP .BI "--s3-european-buckets" +.RS When using the Amazon S3 backend, create buckets in Europe instead of the default (requires -.BI --s3-use-new-style +.B --s3-use-new-style ). Also see the .B EUROPEAN S3 BUCKETS section. -This option does not apply when using the newer boto3 backend, which -does not create buckets. +.B NOTE: +This option does not apply when using the +.B boto3 +backend, which does not create buckets. See also .B "A NOTE ON AMAZON S3" below. +.RE .TP -.BI "--s3-unencrypted-connection" -Don't use SSL for connections to S3. - -This may be much faster, at some cost to confidentiality. +.BI "--s3-multipart-chunk-size" +.RS +Chunk size (in MB, default is 25MB) used for S3 multipart uploads. Make this smaller than +.B --volsize +to maximize the use of your bandwidth. For example, a chunk size of 10MB +with a volsize of 30MB will result in 3 chunks per volume upload. -With this option, anyone who can observe traffic between your computer and S3 -will be able to tell: that you are using Duplicity, the name of the bucket, -your AWS Access Key ID, the increment dates and the amount of data in each -increment. +See also +.B "A NOTE ON AMAZON S3" +below. +.RE -This option affects only the connection, not the GPG encryption of the backup -increment files. Unless that is disabled, an observer will not be able to see -the file names or contents. +.TP +.BI "--s3-multipart-max-procs" +.RS +Specify the maximum number of processes to spawn when performing a multipart +upload to S3. By default, this will choose the number of processors detected +on your system (e.g. 4 for a 4-core system). You can adjust this number as +required to ensure you don't overload your system while maximizing the use of +your bandwidth. -This option is not available when using the newer boto3 backend. +.B NOTE: +This has no effect when using +.B boto3 +backend. See also .B "A NOTE ON AMAZON S3" below. +.RE .TP -.BI "--s3-use-new-style" -When operating on Amazon S3 buckets, use new-style subdomain bucket -addressing. This is now the preferred method to access Amazon S3, but -is not backwards compatible if your bucket name contains upper-case -characters or other characters that are not valid in a hostname. +.BI "--s3-multipart-max-timeout" +.RS +You can control the maximum time (in seconds) a multipart upload can spend on +uploading a single chunk to S3. This may be useful if you find your system +hanging on multipart uploads or if you'd like to control the time variance +when uploading to S3 to ensure you kill connections to slow S3 endpoints. -This option has no effect when using the newer boto3 backend, which -will always use new style subdomain bucket naming. +.B NOTE: +This has no effect when using +.B boto3 +backend. See also .B "A NOTE ON AMAZON S3" below. +.RE .TP -.BI "--s3-use-rrs" -Store volumes using Reduced Redundancy Storage when uploading to Amazon S3. -This will lower the cost of storage but also lower the durability of stored -volumes to 99.99% instead the 99.999999999% durability offered by Standard -Storage on S3. +.BI "--s3-region-name" +.RS +Specifies the region of the S3 storage. + +.B NOTE: +Only in +.B boto3 +backend. +.RE .TP -.BI "--s3-use-ia" -Store volumes using Standard - Infrequent Access when uploading to Amazon S3. -This storage class has a lower storage cost but a higher per-request cost, and -the storage cost is calculated against a 30-day storage minimum. According to -Amazon, this storage is ideal for long-term file storage, backups, and disaster -recovery. +.BI "--s3-unencrypted-connection" +.RS +Disable SSL for connections to S3. This may be much faster, at some cost to confidentiality. + +With this option set, anyone between your computer and S3 can observe the traffic +and will be able to tell: that you are using Duplicity, the name of the bucket, +your AWS Access Key ID, the increment dates and the amount of data in each increment. + +This option affects only the connection, not the GPG encryption of the backup +increment files. Unless that is disabled, an observer will not be able to see +the file names or contents. + +See also +.B "A NOTE ON AMAZON S3" +below. +.RE .TP -.BI "--s3-use-onezone-ia" -Store volumes using One Zone - Infrequent Access when uploading to Amazon S3. -This storage is similar to Standard - Infrequent Access, but only stores object -data in one Availability Zone. +.BI "--s3-use-deep-archive" +.RS +Store volumes using Glacier Deep Archive S3 when uploading to Amazon S3. This storage class +has a lower cost of storage but a higher per-request cost along with delays +of up to 48 hours from the time of retrieval request. This storage cost is +calculated against a 180-day storage minimum. According to Amazon this storage is +ideal for data archiving and long-term backup offering 99.999999999% durability. +To restore a backup you will have to manually migrate all data stored on AWS +Glacier Deep Archive back to Standard S3 and wait for AWS to complete the migration. + +.B NOTE: +Duplicity will store the manifest.gpg files from full and incremental backups on +AWS S3 standard storage to allow quick retrieval for later incremental backups, +all other data is stored in S3 Glacier Deep Archive. +.RE .TP .BI "--s3-use-glacier" +.RS Store volumes using Glacier Flexible Storage when uploading to Amazon S3. This storage class has a lower cost of storage but a higher per-request cost along with delays of up to 12 hours from the time of retrieval request. This storage cost is @@ -817,127 +927,96 @@ ideal for data archiving and long-term backup offering 99.999999999% durability. To restore a backup you will have to manually migrate all data stored on AWS Glacier back to Standard S3 and wait for AWS to complete the migration. -.B Notice: + +.B NOTE: Duplicity will store the manifest.gpg files from full and incremental backups on AWS S3 standard storage to allow quick retrieval for later incremental backups, all other data is stored in S3 Glacier. +.RE .TP .BI "--s3-use-glacier-ir" +.RS Store volumes using Glacier Instant Retrieval when uploading to Amazon S3. This storage class is similar to Glacier Flexible Storage but offers instant retrieval at standard speeds. -.B Notice: + +.B NOTE: Duplicity will store the manifest.gpg files from full and incremental backups on AWS S3 standard storage to allow quick retrieval for later incremental backups, all other data is stored in S3 Glacier. +.RE .TP -.BI "--s3-use-deep-archive" -Store volumes using Glacier Deep Archive S3 when uploading to Amazon S3. This storage class -has a lower cost of storage but a higher per-request cost along with delays -of up to 48 hours from the time of retrieval request. This storage cost is -calculated against a 180-day storage minimum. According to Amazon this storage is -ideal for data archiving and long-term backup offering 99.999999999% durability. -To restore a backup you will have to manually migrate all data stored on AWS -Glacier Deep Archive back to Standard S3 and wait for AWS to complete the migration. -.B Notice: -Duplicity will store the manifest.gpg files from full and incremental backups on -AWS S3 standard storage to allow quick retrieval for later incremental backups, -all other data is stored in S3 Glacier Deep Archive. - -Glacier Deep Archive is only available when using the newer boto3 backend. +.BI "--s3-use-ia" +Store volumes using Standard - Infrequent Access when uploading to Amazon S3. +This storage class has a lower storage cost but a higher per-request cost, and +the storage cost is calculated against a 30-day storage minimum. According to +Amazon, this storage is ideal for long-term file storage, backups, and disaster +recovery. .TP .BI "--s3-use-multiprocessing" +.RS Allow multipart volumne uploads to S3 through multiprocessing. This option requires Python 2.6 and can be used to make uploads to S3 more efficient. If enabled, files duplicity uploads to S3 will be split into chunks and uploaded in parallel. Useful if you want to saturate your bandwidth or if large files are failing during upload. -This has no effect when using the newer boto3 backend. Boto3 always -attempts to multiprocessing when it is believed it will be more efficient. - -See also -.B "A NOTE ON AMAZON S3" -below. - -.TP -.BI "--s3-use-server-side-encryption" -Allow use of server side encryption in S3 - -.TP -.BI "--s3-multipart-chunk-size" -Chunk size (in MB) used for S3 multipart uploads. Make this smaller than -.BI --volsize -to maximize the use of your bandwidth. For example, a chunk size of 10MB -with a volsize of 30MB will result in 3 chunks per volume upload. - -See also -.B "A NOTE ON AMAZON S3" -below. - -.TP -.BI "--s3-multipart-max-procs" -Specify the maximum number of processes to spawn when performing a multipart -upload to S3. By default, this will choose the number of processors detected -on your system (e.g. 4 for a 4-core system). You can adjust this number as -required to ensure you don't overload your system while maximizing the use of -your bandwidth. - -This has no effect when using the newer boto3 backend. +.B NOTE: +This has no effect when using the +.B boto3 +backend. Boto3 always attempts to use multiprocessing. See also .B "A NOTE ON AMAZON S3" below. +.RE .TP -.BI "--s3-multipart-max-timeout" -You can control the maximum time (in seconds) a multipart upload can spend on -uploading a single chunk to S3. This may be useful if you find your system -hanging on multipart uploads or if you'd like to control the time variance -when uploading to S3 to ensure you kill connections to slow S3 endpoints. +.BI "--s3-use-new-style" +.RS +When operating on Amazon S3 buckets, use new-style subdomain bucket +addressing. This is now the preferred method to access Amazon S3, but +is not backwards compatible if your bucket name contains upper-case +characters or other characters that are not valid in a hostname. -This has no effect when using the newer boto3 backend. +.B NOTE: +This option has no effect when using the +.B boto3 +backend, which will always use new style subdomain bucket naming. See also .B "A NOTE ON AMAZON S3" below. +.RE .TP -.BI "--s3-region-name" -Specifies the region of the S3 storage. - -This is currently only used in the newer boto3 backend. +.BI "--s3-use-onezone-ia" +Store volumes using One Zone - Infrequent Access when uploading to Amazon S3. +This storage is similar to Standard - Infrequent Access, but only stores object +data in one Availability Zone. .TP -.BI "--s3-endpoint-url" -Specifies the endpoint URL of the S3 storage. - -This is currently only used in the newer boto3 backend. +.BI "--s3-use-rrs" +Store volumes using Reduced Redundancy Storage when uploading to Amazon S3. +This will lower the cost of storage but also lower the durability of stored +volumes to 99.99% instead the 99.999999999% durability offered by Standard +Storage on S3. .TP -.BI "--azure-blob-tier" -Standard storage tier used for backup files (Hot|Cool|Archive). +.BI "--s3-use-server-side-encryption" +Allow use of server side encryption in S3 .TP -.BI "--azure-max-single-put-size" -Specify the number of the largest supported upload size where the Azure -library makes only one put call. If the content size is known and below this -value the Azure library will only perform one put request to upload one block. -The number is expected to be in bytes. - +.B --s3-use-server-side-kms-encryption +.PD 0 .TP -.BI "--azure-max-block-size" -Specify the number for the block size used by the Azure library to upload -blobs if it is split into multiple blocks. -The maximum block size the service supports is 104857600 (100MiB) and the -default is 4194304 (4MiB) - +.BI "--s3-kms-key-id " key_id +.PD 0 .TP -.BI "--azure-max-connections" -Specify the number of maximum connections to transfer one blob to Azure -blob size exceeds 64MB. The default values is 2. +.BI "--s3-kms-grant " grant +Enable server-side encryption using key management service. .TP .BI "--scp-command " command @@ -996,6 +1075,7 @@ .TP .BI "--ssh-options " options +.RS Allows you to pass options to the ssh backend. Can be specified multiple times or as a space separated options list. The @@ -1016,7 +1096,7 @@ .B NOTE: The -.I "ssh paramiko backend" +.B "ssh paramiko backend" currently supports only the .B -i or @@ -1027,57 +1107,70 @@ .B -oGlobalKnownHostsFile settings. If needed provide more host specific options via ssh_config file. .RE +.RE .TP .BI "--ssl-cacert-file " file +.RS .B (only webdav & lftp backend) Provide a cacert file for ssl certificate verification. -.br + See also .BR "A NOTE ON SSL CERTIFICATE VERIFICATION" . +.RE .TP .BI "--ssl-cacert-path " path/to/certs/ +.RS .B (only webdav backend and python 2.7.9+ OR lftp+webdavs and a recent lftp) Provide a path to a folder containing cacert files for ssl certificate verification. -.br + See also .BR "A NOTE ON SSL CERTIFICATE VERIFICATION" . +.RE .TP .BI --ssl-no-check-certificate +.RS .B (only webdav & lftp backend) Disable ssl certificate verification. -.br + See also .BR "A NOTE ON SSL CERTIFICATE VERIFICATION" . +.RE .TP .BI --swift-storage-policy +.RS Use this storage policy when operating on Swift containers. -.br + See also .BR "A NOTE ON SWIFT (OPENSTACK OBJECT STORAGE) ACCESS" . +.RE .TP .BI "--metadata-sync-mode " mode +.RS This option defaults to 'partial', but you can set it to 'full' -.br + Use 'partial' to avoid syncing metadata for backup chains that you are not going to use. This saves time when restoring for the first time, and lets you restore an old backup that was encrypted with a different passphrase by supplying only the target passphrase. -.br + Use 'full' to sync metadata for all backup chains on the remote. +.RE .TP .BI "--tempdir " directory +.RS Use this existing directory for duplicity temporary files instead of the system default, which is usually the /tmp directory. This option supersedes any environment variable. -.br + See also .BR "ENVIRONMENT VARIABLES" . +.RE .TP .BI -t time ", --time " time ", --restore-time " time @@ -1098,6 +1191,7 @@ .TP .BI --use-agent +.RS If this option is specified, then .I --use-agent is passed to the GnuPG encryption process and it will try to connect to @@ -1107,8 +1201,8 @@ or .I --sign-key if needed. -.br -.B Note: + +.B NOTE: Contrary to previous versions of duplicity, this option will also be honored by GnuPG 2 and newer versions. If GnuPG 2 is in use, duplicity passes the option .I --pinentry-mode=loopback @@ -1118,24 +1212,28 @@ uses the agent only if .I --use-agent is given, just like GnuPG 1. +.RE .TP .BI "--verbosity " level ", -v" level +.RS Specify output verbosity level (log level). Named levels and corresponding values are 0 Error, 2 Warning, 4 Notice (default), 8 Info, 9 Debug (noisiest). .br .I level may also be -.br +.RS .B a character: e, w, n, i, d .br .B a word: error, warning, notice, info, debug +.RE The options -v4, -vn and -vnotice are functionally equivalent, as are the mixed/\ upper-case versions -vN, -vNotice and -vNOTICE. +.RE .TP .BI --version @@ -1147,6 +1245,17 @@ .IR number MB. Default is 200MB. +.TP +.BI "--webdav-headers " "csv formatted key,value pairs" +.RS +The input format is comma separated list of key,value pairs. Standard +CSV encoding may be used. + +For example to set a Cookie use 'Cookie,name=value', or '"Cookie","name=value"'. + +You can set multiple headers, e.g. '"Cookie","name=value","Authorization","xxx"'. +.RE + .SH ENVIRONMENT VARIABLES .TP @@ -1181,232 +1290,299 @@ .SH URL FORMAT Duplicity uses the URL format (as standard as possible) to define data locations. +Major difference is that the whole host section is optional for some backends. +.br +.B NOTE: +If path starts with an extra '/' it usually denotes an absolute path on the backend. + The generic format for a URL is: -.PP + .RS -scheme://[user[:password]@]host[:port]/[/]path -.PP +scheme://[[user[:password]@]host[:port]/][/]path +.RE + +or + +.RS +scheme://[/]path .RE + It is not recommended to expose the password on the command line since it could be revealed to anyone with permissions to do process listings, it is permitted however. Consider setting the environment variable .B FTP_PASSWORD instead, which is used by most, if not all backends, regardless of it's name. -.PP + In protocols that support it, the path may be preceded by a single slash, '/path', to represent a relative path to the target home directory, or preceded by a double slash, '//path', to represent an absolute filesystem path. -.PP -.B Note: -.RS + +.B NOTE: Scheme (protocol) access may be provided by more than one backend. -In case the default backend is buggy or simply not working in a specific case it might be worth trying an alternative implementation. -Alternative backends can be selected by prefixing the scheme with the name of the alternative backend e.g. -.B ncftp+ftp:// +In case the default backend is buggy or simply not working in a specific case +it might be worth trying an alternative implementation. Alternative backends +can be selected by prefixing the scheme with the name of the alternative +.RB "backend e.g. " ncftp+ftp:// and are mentioned below the scheme's syntax summary. -.RE -.PP Formats of each of the URL schemes follow: -.PP +.TP .B "Amazon Drive Backend" -.PP .RS ad://some_dir -.PP + See also .B "A NOTE ON AMAZON DRIVE" .RE -.PP + +.TP .BR "Azure" -.PP .RS azure://container-name -.PP + See also .B "A NOTE ON AZURE ACCESS" .RE -.PP + +.TP .BR "B2" -.PP .RS b2://account_id[:application_key]@bucket_name/[folder/] .RE -.PP + +.TP .BR "Box" -.PP .RS box:///some_dir[?config=path_to_config] -.PP + See also .B "A NOTE ON BOX ACCESS" .RE -.PP + +.TP .BR "Cloud Files" " (Rackspace)" -.PP .RS cf+http://container_name -.PP + See also .B "A NOTE ON CLOUD FILES ACCESS" .RE -.PP + +.TP .B Dropbox -.PP .RS dpbx:///some_dir -.PP + Make sure to read .BR "A NOTE ON DROPBOX ACCESS" " first!" .RE -.PP -.B "Local file path" -.PP + +.TP +.BR File " (local file system)" .RS file://[relative|/absolute]/local/path .RE -.PP + +.TP .BR "FISH" " (Files transferred over Shell protocol) over ssh" -.PP .RS fish://user[:password]@other.host[:port]/[relative|/absolute]_path .RE -.PP + +.TP .B "FTP" -.PP .RS ftp[s]://user[:password]@other.host[:port]/some_dir -.PP + .B NOTE: use lftp+, ncftp+ prefixes to enforce a specific backend, default is lftp+ftp://... .RE -.PP + +.TP +.BR "Google Cloud Storage" " (GCS via Interoperable Access) +.RS +s3://bucket[/path] + +.B NOTE: +use boto+gs://bucket[/path] or boto+s3://bucket[/path] to use legacy +.B boto +backend. default is boto3+s3:// + +See +.B "A NOTE ON GOOGLE CLOUD STORAGE" +about +.B needed +endpoint option and env vars for authentification. +.RE + +.TP .B "Google Docs" -.PP .RS gdocs://user[:password]@other.host/some_dir -.PP + .B NOTE: use pydrive+, gdata+ prefixes to enforce a specific backend, default is pydrive+gdocs://... .RE -.PP -.B "Google Cloud Storage" -.PP + +.TP +.B "Google Drive" + .RS -gs://bucket[/prefix] +gdrive://@developer.gserviceaccount.com/some_dir + +See also +.B "A NOTE ON GDRIVE BACKEND" +below. .RE -.PP + +.TP .B "HSI" -.PP .RS hsi://user[:password]@other.host/some_dir .RE -.PP + +.TP .B "hubiC" -.PP .RS cf+hubic://container_name -.PP + See also .B "A NOTE ON HUBIC" .RE -.PP + +.TP .B "IMAP email storage" -.PP .RS imap[s]://user[:password]@host.com[/from_address_prefix] -.PP + See also .B "A NOTE ON IMAP" .RE -.PP -.B "MEGA.nz cloud storage (only works for accounts created prior to November 2018, uses ""megatools"")" -.PP + +.TP +.BR "MediaFire" +.RS +mf://user[:password]@mediafire.com/some_dir + +See also +.B "A NOTE ON MEDIAFIRE BACKEND" +below. +.RE + +.TP +.BR "MEGA.nz cloud storage" " (only works for accounts created prior to November 2018, uses ""megatools"")" .RS mega://user[:password]@mega.nz/some_dir -.PP + .B NOTE: if not given in the URL, relies on password being stored within $HOME/.megarc (as used by the "megatools" utilities) .RE -.PP -.B "MEGA.nz cloud storage (works for all MEGA accounts, uses ""MEGAcmd"" tools)" -.PP + +.TP +.BR "MEGA.nz cloud storage" " (works for all MEGA accounts, uses ""MEGAcmd"" tools)" .RS megav2://user[:password]@mega.nz/some_dir megav3://user[:password]@mega.nz/some_dir[?no_logout=1] (For latest MEGAcmd) -.PP + .B NOTE: despite "MEGAcmd" no longer uses a configuration file, for convenience storing the user password this backend searches it in the $HOME/.megav2rc file (same syntax as the old $HOME/.megarc) [Login] Username = MEGA_USERNAME Password = MEGA_PASSWORD .RE -.PP + +.TP +.BR "multi" +.RS +multi:///path/to/config.json + +See also +.B "A NOTE ON MULTI BACKEND" +below. +.RE + +.TP .B "OneDrive Backend" -.PP .RS onedrive://some_dir .RE -.PP + +.TP .B "Par2 Wrapper Backend" -.PP .RS par2+scheme://[user[:password]@]host[:port]/[/]path -.PP + See also .B "A NOTE ON PAR2 WRAPPER BACKEND" .RE -.PP + +.TP +.BR "Public Cloud Archive" " (OVH)" +.RS +pca://container_name[/prefix] + +See also +.B "A NOTE ON PCA ACCESS" +.RE + +.TP +.BR "pydrive" +.RS +pydrive://@developer.gserviceaccount.com/some_dir + +See also +.B "A NOTE ON PYDRIVE BACKEND" +below. +.RE + +.TP .B "Rclone Backend" -.PP .RS rclone://remote:/some_dir .RE -.PP + See also .B "A NOTE ON RCLONE BACKEND" .RE -.PP + +.TP .B "Rsync via daemon" -.PP .RS rsync://user[:password]@host.com[:port]::[/]module/some_dir -.PP .RE -.B "Rsync over ssh (only key auth)" -.PP + +.TP +.BR "Rsync over ssh" " (only key auth)" .RS rsync://user@host.com[:port]/[relative|/absolute]_path .RE -.PP + +.TP .BR "S3 storage" " (Amazon)" -.PP .RS -s3://host[:port]/bucket_name[/prefix] -.br -s3+http://bucket_name[/prefix] -.br -.BR "defaults" " to the legacy boto backend based on boto v2 (last update 2018/07)" +s3:///bucket_name[/path] + +.BR "defaults" " to the" +.BR boto3 " backend boto3+s3://" .br -.BR "alternatively" " try the newer boto3+s3://bucket_name[/prefix]" -.PP +.BR "alternatively" " try the legacy" +.BR boto " backend boto+s3://host[:port]/bucket_name[/path]" + For details see .B "A NOTE ON AMAZON S3" -and see also -.B "A NOTE ON EUROPEAN S3 BUCKETS" below. .RE -.PP -.B "SCP/SFTP access" -.PP + +.TP +.BR "SCP/SFTP" " Secure Copy Protocol/SSH File Transfer Protocol" .RS scp://.. or .br sftp://user[:password]@other.host[:port]/[relative|/absolute]_path -.PP + .BR "defaults" " are paramiko+scp:// and paramiko+sftp://" .br .BR "alternatively" " try pexpect+scp://, pexpect+sftp://, lftp+sftp://" @@ -1417,88 +1593,39 @@ and .BR "A NOTE ON SSH BACKENDS" . .RE -.PP -.BR "Swift" " (Openstack)" -.PP + +.TP +.BR "slate" .RS -swift://container_name[/prefix] -.PP +slate://[slate-id] + See also -.B "A NOTE ON SWIFT (OPENSTACK OBJECT STORAGE) ACCESS" +.B "A NOTE ON SLATE BACKEND" .RE -.PP -.BR "Public Cloud Archive" " (OVH)" -.PP + +.TP +.BR "Swift" " (Openstack)" .RS -pca://container_name[/prefix] -.PP +swift://container_name[/prefix] + See also -.B "A NOTE ON PCA ACCESS" +.B "A NOTE ON SWIFT (OPENSTACK OBJECT STORAGE) ACCESS" .RE -.PP + +.TP .B "Tahoe-LAFS" -.PP .RS tahoe://alias/directory .RE -.PP + +.TP .B "WebDAV" -.PP .RS webdav[s]://user[:password]@other.host[:port]/some_dir -.PP + .B alternatively try lftp+webdav[s]:// .RE -.PP -.BR "slate" -.PP -.RS -slate://[slate-id] -.PP -See also -.B "A NOTE ON SLATE BACKEND" -.RE -.PP -.BR "pydrive" -.PP -.RS -pydrive://@developer.gserviceaccount.com/some_dir -.PP -See also -.B "A NOTE ON PYDRIVE BACKEND" -below. -.RE -.PP -.BR "gdrive" -.PP -.RS -gdrive://@developer.gserviceaccount.com/some_dir -.PP -See also -.B "A NOTE ON GDRIVE BACKEND" -below. -.RE -.PP -.BR "multi" -.PP -.RS -multi:///path/to/config.json -.PP -See also -.B "A NOTE ON MULTI BACKEND" -below. -.RE -.PP -.BR "MediaFire" -.PP -.RS -mf://user[:password]@mediafire.com/some_dir -.PP -See also -.B "A NOTE ON MEDIAFIRE BACKEND" -below. -.RE .SH TIME FORMATS duplicity uses time strings in two places. Firstly, many of the files @@ -1682,7 +1809,7 @@ The .BR --include-filelist , and -.BI --exclude-filelist , +.BR --exclude-filelist , options also introduce file selection conditions. They direct duplicity to read in a text file (either ASCII or UTF-8), each line of which is a file specification, and to include or exclude the @@ -1787,38 +1914,83 @@ .SH A NOTE ON AMAZON S3 When backing up to Amazon S3, two backend implementations are available. -The schemes "s3" and "s3+http" are implemented using the older boto library, -which has been deprecated and is no longer supported. The "boto3+s3" scheme -is based on the newer boto3 library. This new backend fixes several known -limitations in the older backend, which have crept in as -Amazon S3 has evolved while the deprecated boto library has not kept up. - -The boto3 backend should behave largely the same as the older S3 backend, -but there are some differences in the handling of some of the "S3" options. -Additionally, there are some compatibility differences with the new backed. -Because of these reasons, both backends have been retained for the time being. -See the documentation for specific options regarding differences related to +The older +.B boto +library, which is deprecated and is no longer maintained. +And the recent +.B boto3 +backend based on the newer boto3 library. The new backend fixes several known +limitations in the older backend, which developed as Amazon S3 evolved. + +The +.B boto3 +backend should behave largely the same as the older backend, +but there are some differences in the supported "--s3-..." options. +Additionally, there are some compatibility differences. +.br +See the documentation of each option above regarding differences related to each backend. -The boto3 backend does not support bucket creation. -This is a deliberate choice which simplifies the code, and side steps +The +.BR boto3 " backend does not support bucket creation." +This deliberate choice simplifies the code, and side steps problems related to region selection. Additionally, it is probably not a good practice to give your backup role bucket creation rights. In most cases the role used for backups should probably be limited to specific buckets. -The boto3 backend only supports newer domain style buckets. Amazon is moving +The +.B boto3 +backend only supports newer domain style buckets. Amazon is moving to deprecate the older bucket style, so migration is recommended. -Use the older s3 backend for compatibility with backups stored in -buckets using older naming conventions. +Use the +.B boto +backend for compatibility with buckets using older naming conventions. -The boto3 backend does not currently support initiating restores +The +.B boto3 +backend does not currently support initiating restores from the glacier storage class. When restoring a backup from glacier or glacier deep archive, the backup files must first be restored out of band. There are multiple options when restoring backups from cold storage, which vary in both cost and speed. See Amazon's documentation for details. +Both backends use environment variables for authentification: +.br +.RS +.BR AWS_ACCESS_KEY_ID " (required)," +.br +.BR AWS_SECRET_ACCESS_KEY " (required)" +.br +or +.br +.BR BOTO_CONFIG " (required) pointing to a boto config file." +.RE +For simplicity's sake we will document the use of the AWS_* vars only. +Research boto documentation available in the web if you want to use the config file. + +.BR "boto3 backend" " example backup command line:" + +.RS +AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= duplicity /some/path s3:///bucket/subfolder +.RE + +.RB "you may add " "--s3-endpoint-url" +.RB "and " "--s3-region-name" +and other s3 options documented above if needed. + +legacy +.BR "boto backend" " example backup command line:" + +.RS +AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= duplicity /some/path boto+s3://[host:port]/bucket/subfolder +.RE + +.RB "The url host setting is optional and allows to define a custom endpoint host. you may add " "--s3-european-buckets" +and other s3 options documented above if needed. + + .SH A NOTE ON AZURE ACCESS The Azure backend requires the Microsoft Azure Storage Blobs client library for Python to be installed on the system. @@ -2002,36 +2174,170 @@ or HTTP errors when trying to upload files to your newly created bucket. Give it a few minutes and the bucket should function normally. -.SH A NOTE ON FILENAME PREFIXES +.SH A NOTE ON FILENAME PREFIXES + +Filename prefixes can be used in +.B "multi backend " +with +.B "mirror " +mode to define affinity rules. They can also be used in conjunction with +S3 lifecycle rules to transition archive files to Glacier, while keeping +metadata (signature and manifest files) on S3. + +Duplicity does not require access to archive files except when restoring from backup. + +.SH A NOTE ON GOOGLE CLOUD STORAGE (GCS via Interoperable Access) +.SS Overview +Duplicity access to GCS currently relies on it's Interoperability API (basically S3 for GCS). +This needs to actively be enabled before access is possible. For details read the next section +.B Preparations +below. +.PP +Two backends are available to access S3 namely +.B boto3 +which is used via +.I s3:// +(alias for +.I boto3+s3:// +) and the legacy +.B boto +backend, usable via +.IR boto+s3:// . + +.SS Preparations +.IP 1. +login on https://console.cloud.google.com/ +.IP 2. +go to +.B Cloud Storage->Settings->Interoperability +.IP 3. +create a +.BR "Service account" " (if needed)" +.IP 4. +create +.B Service account HMAC +access key and secret +.RB ( "!!instantly copy!!" +the secret, it can NOT be recovered later) +.IP 5. +go to +.B Cloud Storage->Browser +.IP 6. +create a bucket +.IP 7. +add permissions for +.B Service account +that was used to set up Interoperability access above + +.PP +Once set up you can use the generated Interoperable Storage Access key and secret and pass them to duplicity as described in the next section. + +.SS Usage +The following examples show accessing GCS via S3 for a collection-status action. +The shown env vars, options and url format can be applied for all other actions as well of course. + +using +.B boto3 +supplying the +.B --s3-endpoint-url +manually. + +.RS + AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= duplicity collection-status s3:/// --s3-endpoint-url=https://storage.googleapis.com +.RE + +.RB "or alternatively with legacy " boto +using either +.IR boto+gs:// . + +.RS +GS_ACCESS_KEY_ID= GS_SECRET_ACCESS_KEY= duplicity collection-status boto+gs:/// + +.BR NOTE: " The auth env vars are prefixed" +.B GS_ +in this case! +.RE + +or +.I boto+s3:// +supplying the +.B --s3-endpoint-url +manually. + +.RS + AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= duplicity collection-status s3:/// --s3-endpoint-url=https://storage.googleapis.com +.RE + +Alternatively, you can run +.B "gsutil config -a" +to have the Google Cloud Storage utility populate the +.B ~/.boto +configuration file. + +.B NOTE: +Also see section +.B URL FORMAT +for a brief overview about the url format expected. + +.SH A NOTE ON GDRIVE BACKEND +GDrive: is a rewritten PyDrive: backend with less dependencies, and a +simpler setup - it uses the JSON keys downloaded directly from Google +Cloud Console. + +Note Google has 2 drive methods, `Shared(previously Team) Drives` and `My Drive`, +both can be shared but require different addressing + +.B For a Google Shared Drives folder + +Share Drive ID specified as a query parameter, driveID, in the backend URL. +Example: + gdrive://developer.gserviceaccount.com/target-folder/?driveID= + +.B For a Google My Drive based shared folder + +MyDrive folder ID specified as a query parameter, myDriveFolderID, in the backend URL +Example + export GOOGLE_SERVICE_ACCOUNT_URL=@.iam.gserviceaccount.com + gdrive://${GOOGLE_SERVICE_ACCOUNT_URL}/?myDriveFolderID= + + +There are also two ways to authenticate to use GDrive: with a regular account or with a +"service account". With a service account, a separate account is +created, that is only accessible with Google APIs and not a web login. +With a regular account, you can store backups in your normal Google +Drive. + +To use a service account, go to the Google developers console at +https://console.developers.google.com. Create a project, and make sure +Drive API is enabled for the project. In the "Credentials" section, +click "Create credentials", then select Service Account with JSON key. + +The GOOGLE_SERVICE_JSON_FILE environment variable needs to contain the +path to the JSON file on duplicity invocation. + +export GOOGLE_SERVICE_JSON_FILE= + -Filename prefixes can be used in -.B "multi backend " -with -.B "mirror " -mode to define affinity rules. They can also be used in conjunction with -S3 lifecycle rules to transition archive files to Glacier, while keeping -metadata (signature and manifest files) on S3. +The alternative is to use a regular account. To do this, start as +above, but when creating a new Client ID, select "Create OAuth client +ID", with application type of "Desktop app". Download the +client_secret.json file for the new client, and set the +GOOGLE_CLIENT_SECRET_JSON_FILE environment variable to the path to +this file, and GOOGLE_CREDENTIALS_FILE to a path to a file where +duplicity will keep the authentication token - this location must be +writable. -Duplicity does not require access to archive files except when restoring from backup. +During the first run, you will be prompted to visit an URL in your +browser to grant access to your drive. Once granted, you will receive +a verification code to paste back into Duplicity. The credentials are +then cached in the file references above for future use. -.SH A NOTE ON GOOGLE CLOUD STORAGE -Support for Google Cloud Storage relies on its Interoperable Access, -which must be enabled for your account. Once enabled, you can generate -Interoperable Storage Access Keys and pass them to duplicity via the -.B GS_ACCESS_KEY_ID -and -.B GS_SECRET_ACCESS_KEY -environment variables. Alternatively, you can run -.B "gsutil config -a" -to have the Google Cloud Storage utility populate the -.B ~/.boto -configuration file. -.PP -Enable Interoperable Access: -https://code.google.com/apis/console#:storage -.br -Create Access Keys: -https://code.google.com/apis/console#:storage:legacy +As a sanity check, GDrive checks the host and username from the URL +against the JSON key, and refuses to proceed if the addresses do not +match. Either the email (for the service accounts) or Client ID (for +regular OAuth accounts) must be present in the URL. See +.B URL FORMAT +above. .SH A NOTE ON HUBIC The hubic backend requires the pyrax library to be installed on the system. See @@ -2065,6 +2371,22 @@ .B from_address_prefix will distinguish between different backups. +.SH A NOTE ON MEDIAFIRE BACKEND +This backend requires +.B mediafire +python library to be installed on the system. See +.BR REQUIREMENTS . + +Use URL escaping for username (and password, if provided via command line): + +.PP +.RS +mf://duplicity%40example.com@mediafire.com/some_folder +.PP +.RE + +The destination folder will be created for you if it does not exist. + .SH A NOTE ON MULTI BACKEND The multi backend allows duplicity to combine the storage available in more than one backend store (e.g., you can store across a google drive @@ -2156,6 +2478,39 @@ to adjust the size (and redundancy) of recovery files in .I percent. +.SH A NOTE ON PCA ACCESS +PCA is a long-term data archival solution by OVH. It runs a slightly modified +version of Openstack Swift introducing latency in the data retrieval process. +It is a good pick for a +.BR "multi backend " +configuration where receiving volumes while an other backend is used to store +manifests and signatures. + +.br +The backend requires python-switclient to be installed on the system. +python-keystoneclient is also needed to interact with OpenStack's Keystone +Identity service. +See +.BR REQUIREMENTS . + +It uses following environment variables for authentification: +.BR PCA_USERNAME " (required)," +.BR PCA_PASSWORD " (required)," +.BR PCA_AUTHURL " (required)," +.BR PCA_USERID " (optional)," +.BR PCA_TENANTID " (optional, but either the tenant name or tenant id must be supplied)" +.BR PCA_REGIONNAME " (optional)," +.BR PCA_TENANTNAME " (optional, but either the tenant name or tenant id must be supplied)" + +If the user was previously authenticated, the following environment +variables can be used instead: +.BR PCA_PREAUTHURL " (required)," +.BR PCA_PREAUTHTOKEN " (required)" + +If +.B PCA_AUTHVERSION +is unspecified, it will default to version 2. + .SH A NOTE ON PYDRIVE BACKEND The pydrive backend requires Python PyDrive package to be installed on the system. See .BR REQUIREMENTS . @@ -2211,6 +2566,46 @@ Duplicity. The credentials are then cached in the file references above for future use. +.SH A NOTE ON RCLONE BACKEND +Rclone is a powerful command line program to sync files and directories to and from various cloud storage providers. + +.SS Usage +Once you have configured an rclone remote via + +.RS +rclone config +.RE + +and successfully set up a remote (e.g. gdrive for Google Drive), assuming you can +list your remote files with + +.RS +rclone ls gdrive:mydocuments +.RE + +you can start your backup with + +.RS +duplicity /mydocuments rclone://gdrive:/mydocuments +.RE + +Please note the slash after the second colon. Some storage provider will work with or without slash +after colon, but some other will not. Since duplicity will complain about malformed URL if a slash +is not present, always put it after the colon, and the backend will handle it for you. + +.SS Options +Note that all rclone options can be set by env vars as well. This is properly documented here + +.RS +https://rclone.org/docs/ +.RE + +but in a nutshell you need to take the long option name, strip the leading --, change - to _, make upper case and prepend RCLONE_. for example + +.RS +the equivalent of '--stats 5s' would be the env var RCLONE_STATS=5s +.RE + .SH A NOTE ON SLATE BACKEND Three environment variables are used with the slate backend: 1. @@ -2236,92 +2631,6 @@ Here's a demo: https://gitlab.com/Shr1ftyy/duplicity/uploads/675664ef0eb431d14c8e20045e3fafb6/slate_demo.mp4 -.SH A NOTE ON GDRIVE BACKEND -GDrive: is a rewritten PyDrive: backend with less dependencies, and a -simpler setup - it uses the JSON keys downloaded directly from Google -Cloud Console. - -Note Google has 2 drive methods, `Shared(previously Team) Drives` and `My Drive`, -both can be shared but require different addressing - -.B For a Google Shared Drives folder - -Share Drive ID specified as a query parameter, driveID, in the backend URL. -Example: - gdrive://developer.gserviceaccount.com/target-folder/?driveID= - -.B For a Google My Drive based shared folder - -MyDrive folder ID specified as a query parameter, myDriveFolderID, in the backend URL -Example - export GOOGLE_SERVICE_ACCOUNT_URL=@.iam.gserviceaccount.com - gdrive://${GOOGLE_SERVICE_ACCOUNT_URL}/?myDriveFolderID= - - -There are also two ways to authenticate to use GDrive: with a regular account or with a -"service account". With a service account, a separate account is -created, that is only accessible with Google APIs and not a web login. -With a regular account, you can store backups in your normal Google -Drive. - -To use a service account, go to the Google developers console at -https://console.developers.google.com. Create a project, and make sure -Drive API is enabled for the project. In the "Credentials" section, -click "Create credentials", then select Service Account with JSON key. - -The GOOGLE_SERVICE_JSON_FILE environment variable needs to contain the -path to the JSON file on duplicity invocation. - -export GOOGLE_SERVICE_JSON_FILE= - - -The alternative is to use a regular account. To do this, start as -above, but when creating a new Client ID, select "Create OAuth client -ID", with application type of "Desktop app". Download the -client_secret.json file for the new client, and set the -GOOGLE_CLIENT_SECRET_JSON_FILE environment variable to the path to -this file, and GOOGLE_CREDENTIALS_FILE to a path to a file where -duplicity will keep the authentication token - this location must be -writable. - -During the first run, you will be prompted to visit an URL in your -browser to grant access to your drive. Once granted, you will receive -a verification code to paste back into Duplicity. The credentials are -then cached in the file references above for future use. - -As a sanity check, GDrive checks the host and username from the URL -against the JSON key, and refuses to proceed if the addresses do not -match. Either the email (for the service accounts) or Client ID (for -regular OAuth accounts) must be present in the URL. See -.B URL FORMAT -above. - -.SH A NOTE ON RCLONE BACKEND -Rclone is a powerful command line program to sync files and directories to and from various cloud storage providers. -.PP -Once you have configured an rclone remote via -.PP -.RS -rclone config -.PP -.RE -and successfully set up a remote (e.g. gdrive for Google Drive), assuming you can -list your remote files with -.PP -.RS -rclone ls gdrive:mydocuments -.PP -.RE -you can start your backup with -.PP -.RS -duplicity /mydocuments rclone://gdrive:/mydocuments -.PP -.RE -Please note the slash after the second colon. Some storage provider will work with or without slash -after colon, but some other will not. Since duplicity will complain about malformed URL if a slash -is not present, always put it after the colon, and the backend will handle it for you. - .SH A NOTE ON SSH BACKENDS The .I ssh backends @@ -2379,8 +2688,8 @@ .B SSH lftp backend is simply there because lftp can interact with the ssh cmd line binaries. It is meant as a last resort in case the above options fail for some reason. -.PP -.B Why use sftp instead of scp? + +.SS Why use sftp instead of scp? The change to sftp was made in order to allow the remote system to chroot the backup, thus providing better security and because it does not suffer from shell quoting issues like scp. Scp also does not support any kind of file listing, so sftp or ssh access will always be needed @@ -2454,55 +2763,6 @@ .B SWIFT_AUTHVERSION is unspecified, it will default to version 1. -.SH A NOTE ON PCA ACCESS -PCA is a long-term data archival solution by OVH. It runs a slightly modified -version of Openstack Swift introducing latency in the data retrieval process. -It is a good pick for a -.BR "multi backend " -configuration where receiving volumes while an other backend is used to store -manifests and signatures. - -.br -The backend requires python-switclient to be installed on the system. -python-keystoneclient is also needed to interact with OpenStack's Keystone -Identity service. -See -.BR REQUIREMENTS . - -It uses following environment variables for authentification: -.BR PCA_USERNAME " (required)," -.BR PCA_PASSWORD " (required)," -.BR PCA_AUTHURL " (required)," -.BR PCA_USERID " (optional)," -.BR PCA_TENANTID " (optional, but either the tenant name or tenant id must be supplied)" -.BR PCA_REGIONNAME " (optional)," -.BR PCA_TENANTNAME " (optional, but either the tenant name or tenant id must be supplied)" - -If the user was previously authenticated, the following environment -variables can be used instead: -.BR PCA_PREAUTHURL " (required)," -.BR PCA_PREAUTHTOKEN " (required)" - -If -.B PCA_AUTHVERSION -is unspecified, it will default to version 2. - -.SH A NOTE ON MEDIAFIRE BACKEND -This backend requires -.B mediafire -python library to be installed on the system. See -.BR REQUIREMENTS . - -Use URL escaping for username (and password, if provided via command line): - -.PP -.RS -mf://duplicity%40example.com@mediafire.com/some_folder -.PP -.RE - -The destination folder will be created for you if it does not exist. - .SH A NOTE ON SYMMETRIC ENCRYPTION AND SIGNING Signing and symmetrically encrypt at the same time with the gpg binary on the command line, as used within duplicity, is a specifically challenging issue. @@ -2608,10 +2868,14 @@ .B Microsoft Azure Storage Blobs client library for Python - https://pypi.org/project/azure-storage-blob/ .TP -.BR "boto backend" " (S3 Amazon Web Services, Google Cloud Storage)" -.B boto version 2.0+ +.BR "boto backend" " (S3 Amazon Web Services, Google Cloud Storage) (legacy)" +.B boto version 2.49 (2018/07/11) - http://github.com/boto/boto .TP +.BR "boto3 backend" " (S3 Amazon Web Services, Google Cloud Storage) (default)" +.B boto3 version 1.x +- https://github.com/boto/boto3 +.TP .BR "box backend" " (box.com)" .B boxsdk - https://github.com/box/box-python-sdk @@ -2624,7 +2888,7 @@ .B Dropbox Python SDK - https://www.dropbox.com/developers/reference/sdk .TP -.BR "gdocs gdata backend" " (legacy Google Docs backend)" +.BR "gdocs gdata backend" " (legacy)" .B Google Data APIs Python Client Library - http://code.google.com/p/gdata-python-client/ .TP @@ -2694,7 +2958,7 @@ (Python Cryptography Toolkit) - http://www.dlitz.net/software/pycrypto/ .TP -.B ssh pexpect backend +.BR "ssh pexpect backend" "(legacy)" .B sftp/scp client binaries OpenSSH - http://www.openssh.com/ .br diff -Nru duplicity-0.8.22/CHANGELOG.md duplicity-0.8.23/CHANGELOG.md --- duplicity-0.8.22/CHANGELOG.md 2022-03-04 18:52:39.000000000 +0000 +++ duplicity-0.8.23/CHANGELOG.md 2022-05-15 15:34:02.000000000 +0000 @@ -1,3 +1,84 @@ +## rel.0.8.23 (2022-05-15) + +### New + +* Add --webdav-headers to webdavbackend. Fixes #94. [Kenneth Loafman] + +* Document rclone option setting via env vars. [edeso] + +* Document rclone option setting via env vars. [edeso] + +* Fix other archs, expose rdiffdir. [edeso] + + fix remote build of armhf, arm64, ppc64el arm64 was tested on Debian + 11 arm64 rdiffdir now avail as /snap/bin/duplicity.rdiffdir remove + obsolete folder /usr/lib/python3.9/ in snap + +* Demote boto backend to legacy ... [ede] + + usable via boto+s3:// or boto+gs:// now only removed s3+http:// + scheme added --s3-endpoint-url option as replacement added + --s3-use-deep-archive option changes are document in man page commit + of the same patch set + +* Promote boto3 backend to default s3:// backend ... [ede] + + add --s3-unencrypted-connection support + +* Man page, major sorting, reformatting, S3/GCS documentation update. [ede] + + some updates, added s3 options, clarifications updated Notes on S3 + and Google Cloud Storage usage sort Options, Url Formats, Notes on + alphabetically consistently use "NOTE:" indent properly all over + +### Changes + +* Use cicd image hosted on GitLab. [Kenneth Loafman] + +* Create singular container for testing. [Kenneth Loafman] + +### Fix + +* Fix LP bug #1970124 - obscure error message. [Kenneth Loafman] + + Fixes handling of error message with real path, not temp path. + +### Other + +* Pkg:fix: make extra sure correct python binary is used. [edeso] + + remove unmaintained changelog add shell wrapper(launcher.sh) add + debug script use shell wrapper as snap binary ignores PATH for + python binary on debian + +* Merge remote-tracking branch 'origin/clean-dist' [Kenneth Loafman] + +* Optimize CI/CD to only run when needed. [Kenneth Loafman] + +* Merge remote-tracking branch 'origin/rclone\_options' [Kenneth Loafman] + +* Merge branch 'duplicity-core20' of +git@gitlab.com:duplicity/duplicity.git into duplicity-core20. [Kenneth Loafman] + +* Merge remote-tracking branch 'origin/master' into duplicity-core20. [Kenneth Loafman] + +* Merge branch 'duplicity-core20' into 'master' [Kenneth Loafman] + + upgrade to base core20 + See merge request duplicity/duplicity!80 + +* Upgrade to base core20. [Kenneth Loafman] + +* Merge branch 'website\_links' into 'master' [Kenneth Loafman] + + switch website to gitlab.io, promote duplicity.us + See merge request duplicity/duplicity!79 + +* Switch website to gitlab.io, promote duplicity.us. [ede] + +* Fix website link. [ede] + + ## rel.0.8.22 (2022-03-04) ### New diff -Nru duplicity-0.8.22/debian/changelog duplicity-0.8.23/debian/changelog --- duplicity-0.8.22/debian/changelog 2022-03-04 18:52:39.000000000 +0000 +++ duplicity-0.8.23/debian/changelog 2022-05-15 15:34:02.000000000 +0000 @@ -1,8 +1,8 @@ -duplicity (0.8.22-ppa202203041847~ubuntu22.04.1) jammy; urgency=low +duplicity (0.8.23-ppa202205151528~ubuntu22.04.1) jammy; urgency=low * Auto build. - -- Kenneth Loafman Fri, 04 Mar 2022 18:52:39 +0000 + -- Kenneth Loafman Sun, 15 May 2022 15:34:02 +0000 duplicity (0) UNRELEASED; urgency=low diff -Nru duplicity-0.8.22/debian/git-build-recipe.manifest duplicity-0.8.23/debian/git-build-recipe.manifest --- duplicity-0.8.22/debian/git-build-recipe.manifest 2022-03-04 18:52:39.000000000 +0000 +++ duplicity-0.8.23/debian/git-build-recipe.manifest 2022-05-15 15:34:02.000000000 +0000 @@ -1,2 +1,2 @@ -# git-build-recipe format 0.4 deb-version 0.8.22-ppa202203041847 -lp:duplicity git-commit:23a6e2a824aa0ad7648cf90a1d25009984f0c4c2 +# git-build-recipe format 0.4 deb-version 0.8.23-ppa202205151528 +lp:duplicity git-commit:2917ffe96b500860d3bd9b8b4522500641e5eb1e diff -Nru duplicity-0.8.22/duplicity/backends/_boto_single.py duplicity-0.8.23/duplicity/backends/_boto_single.py --- duplicity-0.8.22/duplicity/backends/_boto_single.py 2022-03-04 18:52:39.000000000 +0000 +++ duplicity-0.8.23/duplicity/backends/_boto_single.py 2022-05-15 15:34:02.000000000 +0000 @@ -133,6 +133,16 @@ def __init__(self, parsed_url): duplicity.backend.Backend.__init__(self, parsed_url) + # parse endpoint + if config.s3_endpoint_url: + import urllib.parse + pu = urllib.parse.urlparse(config.s3_endpoint_url) + if pu.scheme and pu.scheme.lower() == u'http': + config.s3_unencrypted_connection = True + if pu.hostname: + parsed_url.netloc = pu.netloc + parsed_url.hostname = pu.hostname + try: import boto # pylint: disable=import-error from boto.s3.connection import Location @@ -153,8 +163,6 @@ # HC: Caught a socket error, trying to recover raise BackendException(u'Boto requires a bucket name.') - self.scheme = parsed_url.scheme - if self.url_parts: self.key_prefix = u'%s/' % u'/'.join(self.url_parts) else: @@ -165,8 +173,10 @@ # duplicity and boto.storage_uri() have different URI formats. # boto uses scheme://bucket[/name] and specifies hostname on connect() - self.boto_uri_str = u'://'.join((parsed_url.scheme[:2], + self.scheme = duplicity.backend.strip_prefix(parsed_url.scheme, u'boto')[:2] + self.boto_uri_str = u'://'.join((self.scheme, parsed_url.path.lstrip(u'/'))) + if config.s3_european_buckets: self.my_location = Location.EU else: @@ -235,6 +245,8 @@ storage_class = u'GLACIER' elif config.s3_use_glacier_ir and u"manifest" not in remote_filename: storage_class = u'GLACIER_IR' + elif config.s3_use_deep_archive and u"manifest" not in remote_filename: + storage_class = u'DEEP_ARCHIVE' else: storage_class = u'STANDARD' log.Info(u"Uploading %s/%s to %s Storage" % (self.straight_url, remote_filename, storage_class)) diff -Nru duplicity-0.8.22/duplicity/backends/pydrivebackend.py duplicity-0.8.23/duplicity/backends/pydrivebackend.py --- duplicity-0.8.22/duplicity/backends/pydrivebackend.py 2022-03-04 18:52:39.000000000 +0000 +++ duplicity-0.8.23/duplicity/backends/pydrivebackend.py 2022-05-15 15:34:02.000000000 +0000 @@ -83,8 +83,9 @@ account_key, scopes=u'https://www.googleapis.com/auth/drive') else: - signer = crypt.Signer.from_string(account_key) - credentials = ServiceAccountCredentials(parsed_url.username + u'@' + parsed_url.hostname, signer, + signer = crypt.Signer.from_string(account_key) # pylint: disable=used-before-assignment + credentials = ServiceAccountCredentials(parsed_url.username + # pylint: disable=used-before-assignment + u'@' + parsed_url.hostname, signer, scopes=u'https://www.googleapis.com/auth/drive') credentials.authorize(httplib2.Http()) gauth = GoogleAuth(http_timeout=60) diff -Nru duplicity-0.8.22/duplicity/backends/s3_boto3_backend.py duplicity-0.8.23/duplicity/backends/s3_boto3_backend.py --- duplicity-0.8.22/duplicity/backends/s3_boto3_backend.py 2022-03-04 18:52:39.000000000 +0000 +++ duplicity-0.8.23/duplicity/backends/s3_boto3_backend.py 2022-05-15 15:34:02.000000000 +0000 @@ -89,7 +89,9 @@ from botocore.exceptions import ClientError # pylint: disable=import-error self.bucket = None - self.s3 = boto3.resource(u's3', region_name=config.s3_region_name, endpoint_url=config.s3_endpoint_url) + self.s3 = boto3.resource(u's3', region_name=config.s3_region_name, + use_ssl=(not config.s3_unencrypted_connection), + endpoint_url=config.s3_endpoint_url) try: self.s3.meta.client.head_bucket(Bucket=self.bucket_name) @@ -223,4 +225,5 @@ duplicity.backend.register_backend(u"boto3+s3", S3Boto3Backend) -# duplicity.backend.uses_netloc.extend([u'boto3+s3']) +# make boto3 the default s3 backend +duplicity.backend.register_backend(u"s3", S3Boto3Backend) diff -Nru duplicity-0.8.22/duplicity/backends/s3_boto_backend.py duplicity-0.8.23/duplicity/backends/s3_boto_backend.py --- duplicity-0.8.22/duplicity/backends/s3_boto_backend.py 2022-03-04 18:52:39.000000000 +0000 +++ duplicity-0.8.23/duplicity/backends/s3_boto_backend.py 2022-05-15 15:34:02.000000000 +0000 @@ -28,10 +28,6 @@ else: from ._boto_single import BotoBackend -duplicity.backend.register_backend(u"gs", BotoBackend) -duplicity.backend.register_backend(u"s3", BotoBackend) -duplicity.backend.register_backend(u"s3+http", BotoBackend) -duplicity.backend.uses_netloc.extend([u's3']) +duplicity.backend.register_backend(u"boto+gs", BotoBackend) # s3 is also implemented by the newer boto3 backend now duplicity.backend.register_backend(u"boto+s3", BotoBackend) -duplicity.backend.uses_netloc.extend([u'boto+s3']) diff -Nru duplicity-0.8.22/duplicity/backends/webdavbackend.py duplicity-0.8.23/duplicity/backends/webdavbackend.py --- duplicity-0.8.22/duplicity/backends/webdavbackend.py 2022-03-04 18:52:39.000000000 +0000 +++ duplicity-0.8.23/duplicity/backends/webdavbackend.py 2022-05-15 15:34:02.000000000 +0000 @@ -25,6 +25,7 @@ standard_library.install_aliases() from builtins import str from builtins import range + import base64 import http.client import os @@ -143,7 +144,19 @@ u"""Connect to remote store using WebDAV Protocol""" def __init__(self, parsed_url): duplicity.backend.Backend.__init__(self, parsed_url) + self.headers = {u'Connection': u'keep-alive'} + if config.webdav_headers: + try: + self.headers = util.merge_dicts(self.headers, + util.csv_args_to_dict(config.webdav_headers)) + except IndexError as e: + log.FatalError(u"--webdav-headers value has an odd number of arguments. Must be paired.") + except SyntaxError as e: + log.FatalError(u"--webdav-headers value has bad syntax. Check quoting pairs.") + except Exception as e: + log.FatalErrof(u"--webdav-headers value caused error: %s" % e) + self.parsed_url = parsed_url self.digest_challenge = None self.digest_auth_handler = None @@ -152,7 +165,6 @@ self.password = self.get_password() self.directory = self.sanitize_path(parsed_url.path) - log.Info(_(u"Using WebDAV protocol %s") % (config.webdav_proto,)) log.Info(_(u"Using WebDAV host %s port %s") % (parsed_url.hostname, parsed_url.port)) log.Info(_(u"Using WebDAV directory %s") % (self.directory,)) diff -Nru duplicity-0.8.22/duplicity/commandline.py duplicity-0.8.23/duplicity/commandline.py --- duplicity-0.8.22/duplicity/commandline.py 2022-03-04 18:52:39.000000000 +0000 +++ duplicity-0.8.23/duplicity/commandline.py 2022-05-15 15:34:02.000000000 +0000 @@ -634,6 +634,9 @@ parser.add_option(u"--ssl-cacert-path", metavar=_(u"path to a folder with certificate authority files")) parser.add_option(u"--ssl-no-check-certificate", action=u"store_true") + # header options for Webdav + parser.add_option(u"--webdav-headers", metavar=_(u"extra headers for Webdav, like 'Cookie,name=value'")) + # Working directory for the tempfile module. Defaults to /tmp on most systems. parser.add_option(u"--tempdir", dest=u"temproot", type=u"file", metavar=_(u"path")) diff -Nru duplicity-0.8.22/duplicity/config.py duplicity-0.8.23/duplicity/config.py --- duplicity-0.8.22/duplicity/config.py 2022-03-04 18:52:39.000000000 +0000 +++ duplicity-0.8.23/duplicity/config.py 2022-05-15 15:34:02.000000000 +0000 @@ -190,8 +190,8 @@ # FTP data connection type ftp_connection = u'passive' -# Protocol for webdav -webdav_proto = u'http' +# Header options for Webdav +webdav_headers = u"" # Asynchronous put/get concurrency limit # (default of 0 disables asynchronicity). diff -Nru duplicity-0.8.22/duplicity/dup_main.py duplicity-0.8.23/duplicity/dup_main.py --- duplicity-0.8.22/duplicity/dup_main.py 2022-03-04 18:52:39.000000000 +0000 +++ duplicity-0.8.23/duplicity/dup_main.py 2022-05-15 15:34:02.000000000 +0000 @@ -21,7 +21,11 @@ # along with duplicity; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # -# See http://www.nongnu.org/duplicity for more information. +# For more information see +# http://duplicity.us +# or +# http://duplicity.gitlab.io +# . # Please send mail to me or the mailing list if you find bugs or have # any suggestions. @@ -1298,7 +1302,7 @@ def __next__(self): try: res = Block(self.fileobj.read(self.get_read_size())) - except Exception: + except Exception as e: if hasattr(self.fileobj, u'name'): name = self.fileobj.name # name may be a path @@ -1306,8 +1310,7 @@ name = name.name else: name = None - log.FatalError(_(u"Failed to read %s: %s") % - (util.fsdecode(name), sys.exc_info()), + log.FatalError(_(u"Failed to read %s: %s") % (util.fsdecode(fn), util.uexc(e)), log.ErrorCode.generic) if not res.data: self.fileobj.close() diff -Nru duplicity-0.8.22/duplicity/util.py duplicity-0.8.23/duplicity/util.py --- duplicity-0.8.22/duplicity/util.py 2022-03-04 18:52:39.000000000 +0000 +++ duplicity-0.8.23/duplicity/util.py 2022-05-15 15:34:02.000000000 +0000 @@ -31,6 +31,7 @@ from builtins import object from builtins import str +import csv import errno import json import os @@ -39,6 +40,11 @@ import traceback import atexit +if sys.version_info.major == 2: + from cStringIO import StringIO # pylint: disable=import-error +else: + from io import StringIO # pylint: disable=import-error + from duplicity import tarfile import duplicity.config as config import duplicity.log as log @@ -313,3 +319,28 @@ sys.path.insert(0, base) os.environ[u'DEBUG_RUNNING'] = u'yes' + + +def merge_dicts(*dict_args): + u""" + Given any number of dictionaries, shallow copy and merge into a new dict, + precedence goes to key-value pairs in latter dictionaries. + """ + result = {} + for dictionary in dict_args: + result.update(dictionary) + return result + + +def csv_args_to_dict(arg): + u""" + Given the string arg in single line csv format, split into pairs (key, val) + and produce a dictionary from those key:val pairs. + """ + mydict = {} + with StringIO(arg) as infile: + rows = csv.reader(infile) + for row in rows: + for i in range(0, len(row), 2): + mydict[row[i]] = row[i + 1] + return mydict diff -Nru duplicity-0.8.22/.gitlab-ci.yml duplicity-0.8.23/.gitlab-ci.yml --- duplicity-0.8.22/.gitlab-ci.yml 2022-03-04 18:52:39.000000000 +0000 +++ duplicity-0.8.23/.gitlab-ci.yml 2022-05-15 15:34:02.000000000 +0000 @@ -1,9 +1,10 @@ -# Copyright 2019 Nils Tekampe , -# Kenneth Loafman and -# Aaron Whitehouse -# # This file is part of duplicity. # +# Copyright 2022 Nils Tekampe , +# Kenneth Loafman , +# Aaron Whitehouse , +# Edgar Soldin +# # Duplicity is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the # Free Software Foundation; either version 2 of the License, or (at your @@ -22,20 +23,23 @@ # - template: Dependency-Scanning.gitlab-ci.yml .test-template: &test-template - stage: tests - except: - variables: - - $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i - - $SKIP_TESTS + image: registry.gitlab.com/duplicity/duplicity/firstprime/duplicity_test + stage: test + rules: + - if: $CI_COMMIT_MESSAGE =~ /\[skip[ _-]tests?\]/i || $SKIP_TESTS + when: never + - changes: + - bin/duplicity + - bin/rdiffdir + - duplicity/**/* + when: always + - when: never variables: # cache local items PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" # Set to non-interactive so no tzdata prompt DEBIAN_FRONTEND: "noninteractive" - # for upload to pypi.org - TWINE_USERNAME: $TWINE_USERNAME - TWINE_PASSWORD: $TWINE_PASSWORD stages: - qual @@ -43,110 +47,104 @@ - deploy default: - image: ubuntu:18.04 before_script: # Set locale to prevent UTF-8 errors - export LANG=C.UTF-8 - export LC_ALL=C.UTF-8 - - apt-get update - - apt-get install -y 2to3 build-essential git intltool lftp librsync-dev - libffi-dev libssl-dev openssl par2 rdiff tzdata python3-pip - - python3 -m pip install --upgrade pip - - python3 -m pip install --requirement requirements.txt + - export PYENV_ROOT="/root/.pyenv" + - export TOXWORKDIR="/root/.tox" interruptible: true -code_ub18-04: +code: <<: *test-template stage: qual script: - - apt-get install -y python3.8 python3.8-dev - tox -e code - artifacts: - reports: - junit: report.xml -py27_ub18-04: +py27: <<: *test-template stage: test script: - - apt-get install -y python2.7 python2.7-dev - tox -e py27 - artifacts: - reports: - junit: report.xml allow_failure: true -py35_ub16-04: +py35: <<: *test-template stage: test - image: ubuntu:16.04 - before_script: - # Set locale to prevent UTF-8 errors - - export LANG=C.UTF-8 - - export LC_ALL=C.UTF-8 - - apt-get update - - apt-get install -y build-essential git intltool lftp librsync-dev - libffi-dev libssl-dev openssl par2 rdiff tzdata - python3-pip python3.5 python3.5-dev - - python3 -m pip install --upgrade pip==20.3.4 - - python3 -m pip install --requirement requirements.txt script: - - apt-get install -y python3.5 python3.5-dev - tox -e py35 - artifacts: - reports: - junit: report.xml allow_failure: true -py36_ub18-04: +py36: <<: *test-template stage: test script: - - apt-get install -y python3.6 python3.6-dev - tox -e py36 - artifacts: - reports: - junit: report.xml -py37_ub18-04: +py37: <<: *test-template stage: test script: - - apt-get install -y python3.7 python3.7-dev - tox -e py37 - artifacts: - reports: - junit: report.xml -py38_ub18-04: +py38: <<: *test-template stage: test script: - - apt-get install -y python3.8 python3.8-dev - tox -e py38 - artifacts: - reports: - junit: report.xml -py39_ub20-04: +py39: <<: *test-template stage: test + script: + - tox -e py39 + +py310: + <<: *test-template + stage: test + script: + - tox -e py310 + +build_pip: + stage: deploy + when: manual image: ubuntu:20.04 before_script: + - apt-get update + - apt-get install -y git python3-pip git intltool + - python3 -m pip install --upgrade pip + - python3 -m pip install -r requirements.dev + - python3 -m pip install twine + script: + - ./setup.py sdist --dist-dir=. + artifacts: + paths: + - duplicity-*.tar.gz + expire_in: 30 days + +build_snap: + stage: deploy + when: manual + image: ubuntudesktop/gnome-3-38-2004 + before_script: # Set locale to prevent UTF-8 errors - export LANG=C.UTF-8 - export LC_ALL=C.UTF-8 + - export SNAP_ARCH=amd64 + - export SNAPCRAFT_BUILD_INFO=1 - apt-get update - - apt-get install -y build-essential git intltool lftp librsync-dev - libffi-dev libssl-dev openssl par2 rdiff tzdata python3-pip + - apt-get install -y git python3-pip git intltool squashfs-tools - python3 -m pip install --upgrade pip - - python3 -m pip install --requirement requirements.txt + - python3 -m pip install -r requirements.txt -r requirements.dev script: - - apt-get install -y python3.9 python3.9-dev - - tox -e py39 - artifacts: - reports: - junit: report.xml + - tools/makesnap + - tools/installsnap + - tools/testsnap + artifacts: + paths: + - build/duplicity-*/duplicity_*.snap + - build/duplicity-*/duplicity_*.txt + expire_in: 30 days pages: stage: deploy diff -Nru duplicity-0.8.22/README.md duplicity-0.8.23/README.md --- duplicity-0.8.22/README.md 2022-03-04 18:52:39.000000000 +0000 +++ duplicity-0.8.23/README.md 2022-05-15 15:34:02.000000000 +0000 @@ -61,9 +61,13 @@ # HELP -For more information see the duplicity home page at: +For more information see the duplicity web site at: - http://www.nongnu.org/duplicity + http://duplicity.us + + or + + http://duplicity.gitlab.io or post to the mailing list at diff -Nru duplicity-0.8.22/requirements.dev duplicity-0.8.23/requirements.dev --- duplicity-0.8.22/requirements.dev 1970-01-01 00:00:00.000000000 +0000 +++ duplicity-0.8.23/requirements.dev 2022-05-15 15:34:02.000000000 +0000 @@ -0,0 +1,23 @@ +##### testing libraries ##### + + +2to3 +coverage +mock>=3.0.5 +pycodestyle +pylint +pytest +pytest-cov +pytest-runner ; python_version >= '3.6' +setuptools>=44.1.1 +setuptools-scm>=5.0.2 +tox + + +##### documentation libraries ##### + +gitchangelog ; python_version >= '3.6' +myst-parser ; python_version >= '3.6' +pystache ; python_version >= '3.6' +sphinx ; python_version >= '3.6' +sphinx-rtd-theme ; python_version >= '3.6' diff -Nru duplicity-0.8.22/requirements.txt duplicity-0.8.23/requirements.txt --- duplicity-0.8.22/requirements.txt 2022-03-04 18:52:39.000000000 +0000 +++ duplicity-0.8.23/requirements.txt 2022-05-15 15:34:02.000000000 +0000 @@ -2,31 +2,8 @@ fasteners future -python-gettext -setuptools>=44.1.1 -setuptools-scm>=5.0.2 - - -##### testing libraries ##### - - -coverage -mock>=3.0.5 -pycodestyle pydevd ; python_version >= '3.6' -pylint -pytest -pytest-cov -pytest-runner ; python_version >= '3.6' -tox - - -##### documrntation libraries ##### - - -myst-parser ; python_version >= '3.6' -sphinx ; python_version >= '3.6' -sphinx-rtd-theme ; python_version >= '3.6' +python-gettext ##### backend libraries ##### diff -Nru duplicity-0.8.22/setup.py duplicity-0.8.23/setup.py --- duplicity-0.8.22/setup.py 2022-03-04 18:52:39.000000000 +0000 +++ duplicity-0.8.23/setup.py 2022-05-15 15:34:02.000000000 +0000 @@ -53,7 +53,7 @@ from setuptools_scm import get_version # pylint: disable=import-error Version = get_version(**scm_version_args) except Exception as e: - Version = u"0.8.22" + Version = u"0.8.23" print(u"Unable to get SCM version: defaulting to %s" % (Version,)) Reldate = time.strftime(u"%B %d, %Y", time.gmtime(int(os.environ.get(u'SOURCE_DATE_EPOCH', time.time())))) @@ -181,7 +181,18 @@ os.environ[u'COPYFILE_DISABLE'] = u'true' # make the new tarfile and remove tardir - assert not os.system(u"tar czf %s %s" % (tarfile, tardir)) + assert not os.system(u"""tar czf %s \ + --exclude '.*' \ + --exclude Makefile \ + --exclude debian \ + --exclude docs \ + --exclude pylintrc \ + --exclude readthedocs.yaml \ + --exclude testing/docker \ + --exclude testing/manual \ + --exclude tools \ + %s + """ % (tarfile, tardir)) assert not shutil.rmtree(tardir) @@ -268,7 +279,7 @@ author_email=u"ben@emrose.org", maintainer=u"Kenneth Loafman ", maintainer_email=u"kenneth@loafman.com", - url=u"http://duplicity.nongnu.org/index.html", + url=u"http://duplicity.us", python_requires=u">2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4", platforms=[u"any"], packages=[ @@ -291,10 +302,6 @@ ], data_files=get_data_files(), include_package_data=True, - setup_requires=[ - u"setuptools", - u"setuptools_scm", - ], install_requires=[ u"fasteners", u"future", diff -Nru duplicity-0.8.22/snap/local/changelog.txt duplicity-0.8.23/snap/local/changelog.txt --- duplicity-0.8.22/snap/local/changelog.txt 2022-03-04 18:52:39.000000000 +0000 +++ duplicity-0.8.23/snap/local/changelog.txt 1970-01-01 00:00:00.000000000 +0000 @@ -1,12 +0,0 @@ -Snap changelog - -Revision Version Change -6 0.7.19 Added a large number of dependencies beyond those in - requirements.txt based on those imported or used in - backends. -7 0.7.19 Added additional dependencies from .deb package. -8 0.8.00 Added openssh-client, jottalib -9 0.8.00 Changed bin/duplicity to use python2 -10 0.8.01 New duplicity version, no snap changes -11 0.8.02 New duplicity version, no snap changes -12 0.8.03 New duplicity version, no snap changes diff -Nru duplicity-0.8.22/snap/local/debug.sh duplicity-0.8.23/snap/local/debug.sh --- duplicity-0.8.22/snap/local/debug.sh 1970-01-01 00:00:00.000000000 +0000 +++ duplicity-0.8.23/snap/local/debug.sh 2022-05-15 15:34:02.000000000 +0000 @@ -0,0 +1,7 @@ +#!/bin/sh +echo running \'$0\' +echo PATH=\'$PATH\' +echo PYTHONPATH=\'$PYTHONPATH\' +echo python3=\>\'$(which python3)\' +echo python3.8=\>\'$(which python3.8)\' +echo gpg=\>\'$(which gpg)\' diff -Nru duplicity-0.8.22/snap/local/launcher.sh duplicity-0.8.23/snap/local/launcher.sh --- duplicity-0.8.22/snap/local/launcher.sh 1970-01-01 00:00:00.000000000 +0000 +++ duplicity-0.8.23/snap/local/launcher.sh 2022-05-15 15:34:02.000000000 +0000 @@ -0,0 +1,6 @@ +#!/bin/sh +[ -z "$SNAP" ] && { + echo Missing env var SNAP. + exit 1 +} +"$SNAP"/bin/$(basename "${0%.sh}") "$@" diff -Nru duplicity-0.8.22/snap/snapcraft.yaml duplicity-0.8.23/snap/snapcraft.yaml --- duplicity-0.8.22/snap/snapcraft.yaml 2022-03-04 18:52:39.000000000 +0000 +++ duplicity-0.8.23/snap/snapcraft.yaml 2022-05-15 15:34:02.000000000 +0000 @@ -14,17 +14,64 @@ apps: duplicity: - command: bin/duplicity + command: bin/duplicity.sh + environment: + # setup path to find and use packaged python3 in snap + PATH: "$SNAP/usr/sbin:$SNAP/usr/bin:$SNAP/sbin:$SNAP/bin:$PATH" + # fixup pythonpath to find modules, for some reason + # 'lib/python3.8/site-packages' is not added by default + # add two more host paths for users to install modules in + PYTHONPATH: "$SNAP/lib/python3.8/site-packages:\ + $HOME/.local/lib/python3.8/site-packages:\ + /lib/python3.8/site-packages" + rdiffdir: + command: bin/rdiffdir.sh + environment: + # see above + PATH: "$SNAP/usr/sbin:$SNAP/usr/bin:$SNAP/sbin:$SNAP/bin:$PATH" + PYTHONPATH: "$SNAP/lib/python3.8/site-packages:\ + $HOME/.local/lib/python3.8/site-packages:\ + /lib/python3.8/site-packages" + # printing env in snap for debug purposes, help's pinning snap startup issues (missing libs, _rsync ...) + debug: + command: bin/debug.sh + debug2: + command: bin/debug.sh + environment: + # see above + PATH: "$SNAP/usr/sbin:$SNAP/usr/bin:$SNAP/sbin:$SNAP/bin:$PATH" + PYTHONPATH: "$SNAP/lib/python3.8/site-packages:\ + $HOME/.local/lib/python3.8/site-packages:\ + /lib/python3.8/site-packages" parts: duplicity: plugin: python + override-build: | + snapcraftctl build + rm -rfv "$SNAPCRAFT_PART_INSTALL/usr/lib/python3.9" + override-prime: | + snapcraftctl prime + cp -v "$SNAPCRAFT_PROJECT_DIR"/snap/local/launcher.sh "$SNAPCRAFT_PRIME"/bin/duplicity.sh + cp -v "$SNAPCRAFT_PROJECT_DIR"/snap/local/launcher.sh "$SNAPCRAFT_PRIME"/bin/rdiffdir.sh + cp -v "$SNAPCRAFT_PROJECT_DIR"/snap/local/debug.sh "$SNAPCRAFT_PRIME"/bin/ + build-environment: + - PYTHONPATH: "$SNAPCRAFT_PART_INSTALL/usr/lib/python3.8/dist-packages:\ + $SNAPCRAFT_PART_INSTALL/usr/lib/python3/dist-packages:\ + $PYTHONPATH" build-packages: - build-essential - ieee-data - intltool - librsync-dev - python3-dev + # dependencies needed to pip build on non amd64 archs + - libffi-dev + - libxslt1-dev + - libxml2-dev + - rust-all + - libssl-dev + - cargo stage-packages: - gnupg - lftp @@ -35,7 +82,29 @@ - python3-gi - rsync - rclone + - libpython3-stdlib + - libpython3.8-stdlib + - libpython3.8-minimal + - python3-minimal + - python3.8-minimal + - python3-dev + - python3-pip + - python3-setuptools + - python3-distutils + - python3-wheel + - python3-pkg-resources + - python3-venv + # some libraries needed on not amd64 archs + - libicu66 + - libxml2 + - libxslt1.1 python-packages: + # libs needed to avoid some version dependencies. + # use latest version available while still py38. + - grpcio-tools ; python_version == '3.8' + - pbr ; python_version == '3.8' + - requests ; python_version == '3.8' + # normally included libs - azure-storage-blob ; python_version == '3.8' - b2sdk ; python_version == '3.8' - boto ; python_version == '3.8' @@ -58,11 +127,11 @@ - pydrive ; python_version == '3.8' - pyrax ; python_version == '3.8' - python-swiftclient ; python_version == '3.8' - - requests ; python_version == '3.8' - requests-oauthlib ; python_version == '3.8' - setuptools ; python_version == '3.8' - setuptools-scm ; python_version == '3.8' - sx ; python_version == '3.8' - tahoe-lafs ; python_version == '3.8' - urllib3 ; python_version == '3.8' + - wheel ; python_version == '3.8' source: . diff -Nru duplicity-0.8.22/testing/docker/build.sh duplicity-0.8.23/testing/docker/build.sh --- duplicity-0.8.22/testing/docker/build.sh 1970-01-01 00:00:00.000000000 +0000 +++ duplicity-0.8.23/testing/docker/build.sh 2022-05-15 15:34:02.000000000 +0000 @@ -0,0 +1,37 @@ +#!/bin/bash +# +# Copyright 2017 Nils Tekampe +# +# This file is part of duplicity. +# This script sets up a test network for the tests of dupclicity +# This script takes the assumption that the containers for the testinfrastructure do deither run +# or they are removed. It is not intended to have stopped containers. +# +# Duplicity is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. +# +# Duplicity is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with duplicity; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# + +cd `dirname $0` + +export DOCKER_BUILDKIT=0 + +cp ../../requirements.* . +cp ../../setup.py . +cp ../../tox.ini . + +docker build --compress --tag firstprime/duplicity_test --file duplicity_test/Dockerfile . + +rm requirements.* +rm setup.py +rm tox.ini diff -Nru duplicity-0.8.22/testing/docker/docker-compose.yml duplicity-0.8.23/testing/docker/docker-compose.yml --- duplicity-0.8.22/testing/docker/docker-compose.yml 2022-03-04 18:52:39.000000000 +0000 +++ duplicity-0.8.23/testing/docker/docker-compose.yml 2022-05-15 15:34:02.000000000 +0000 @@ -1,5 +1,6 @@ # -# Copyright 2017 Nils Tekampe +# Copyright 2017 Nils Tekampe , +# Kenneth Loafman # # This file is part of duplicity. # This script sets up a test network for the tests of dupclicity @@ -21,54 +22,55 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # -version: "3" - services: - duplicity_test: - build: - context: ../.. - dockerfile: testing/docker/duplicity_test/Dockerfile - image: firstprime/duplicity_test - ports: - - "30000-30009:30000-30009" - entrypoint: /bin/bash - depends_on: - - duplicity_ftp - - duplicity_ssh - stdin_open: true - tty: true - networks: - duplicity_test_net: - ipv4_address: ${DUPLICITY_TEST_SUBNET}.2 - environment: - - DUPLICITY_IP_SSH_SERVER=${DUPLICITY_TEST_SUBNET}.3 - - DUPLICITY_IP_FTP_SERVER=${DUPLICITY_TEST_SUBNET}.4 - volumes: - - ../..:/home/test/duplicity - - - duplicity_ftp: - image: firstprime/duplicity_ftp - ports: - - "21:21" - networks: - duplicity_test_net: - ipv4_address: ${DUPLICITY_TEST_SUBNET}.3 - - - duplicity_ssh: - image: firstprime/duplicity_ssh - ports: - - "2222:22" - tty: true - networks: - duplicity_test_net: - ipv4_address: ${DUPLICITY_TEST_SUBNET}.4 - - -networks: - duplicity_test_net: - ipam: - driver: default - config: - - subnet: ${DUPLICITY_TEST_SUBNET}.0/24 + duplicity_test: + image: firstprime/duplicity_test + container_name: duplicity_test + build: + context: ../.. + dockerfile: testing/docker/duplicity_test/Dockerfile + entrypoint: /bin/bash + stdin_open: true + tty: true + volumes: + - ../..:/root/duplicity + + +# depends_on: +# - duplicity_ftp +# - duplicity_ssh +# ports: +# - "30000-30009:30000-30009" +# networks: +# duplicity_test_net: +# ipv4_address: ${DUPLICITY_TEST_SUBNET}.2 +# environment: +# - DUPLICITY_IP_SSH_SERVER=${DUPLICITY_TEST_SUBNET}.3 +# - DUPLICITY_IP_FTP_SERVER=${DUPLICITY_TEST_SUBNET}.4 + + +# duplicity_ftp: +# image: firstprime/duplicity_ftp +# ports: +# - "21:21" +# networks: +# duplicity_test_net: +# ipv4_address: ${DUPLICITY_TEST_SUBNET}.3 + + +# duplicity_ssh: +# image: firstprime/duplicity_ssh +# ports: +# - "2222:22" +# tty: true +# networks: +# duplicity_test_net: +# ipv4_address: ${DUPLICITY_TEST_SUBNET}.4 + + +# networks: +# duplicity_test_net: +# ipam: +# driver: default +# config: +# - subnet: ${DUPLICITY_TEST_SUBNET}.0/24 diff -Nru duplicity-0.8.22/testing/docker/duplicity_test/Dockerfile duplicity-0.8.23/testing/docker/duplicity_test/Dockerfile --- duplicity-0.8.22/testing/docker/duplicity_test/Dockerfile 2022-03-04 18:52:39.000000000 +0000 +++ duplicity-0.8.23/testing/docker/duplicity_test/Dockerfile 2022-05-15 15:34:02.000000000 +0000 @@ -1,7 +1,8 @@ # -*- Mode:Python; indent-tabs-mode:nil; tab-width:4; encoding:utf8 -*- # # Copyright 2019 Nils Tekampe , -# Kenneth Loafman and Aaron Whitehouse +# Kenneth Loafman , +# Aaron Whitehouse # # This file is part of duplicity. # @@ -24,6 +25,9 @@ # Set locale to prevent UTF-8 errors ENV LANG C.UTF-8 ENV LC_ALL C.UTF-8 +ENV PYENV_ROOT "/root/.pyenv" +ENV PATH "$PYENV_ROOT/bin:$PYENV_ROOT/shims:$PATH" +ENV TOXWORKDIR="/root/.tox" # Set to non-interactive so no tzdata prompt ARG DEBIAN_FRONTEND=noninteractive @@ -32,57 +36,90 @@ # packages needed for testing duplicity RUN apt-get update \ && apt-get install -y \ - 2to3 \ - build-essential \ - git \ - intltool \ - lftp \ - librsync-dev \ - libffi-dev \ - libssl-dev \ - openssl \ - par2 \ - python3-pip \ - python3-dev \ - python3-future \ - python3-pip \ - python3 \ - rdiff \ - tzdata # required for testing/unit/test_statistics.py - + 2to3 \ + build-essential \ + git \ + intltool \ + lftp \ + librsync-dev \ + libffi-dev \ + libssl-dev \ + openssl \ + par2 \ + python3-pip \ + python3 \ + rdiff \ + tzdata \ +# The following packages are for building python via pyenv + && apt-get install -y \ + curl \ + libbz2-dev \ + libffi-dev \ + liblzma-dev \ + libncursesw5-dev \ + libreadline-dev \ + libsqlite3-dev \ + libssl-dev \ + libxml2-dev \ + libxmlsec1-dev \ + llvm \ + make \ + tk-dev \ + wget \ + xz-utils \ + zlib1g-dev \ # The following packages are not necessary for testing but make life easier or support debugging -RUN apt-get install -y \ - ftp \ - iputils-ping \ - mc \ - nano \ - net-tools \ - rsync \ + && apt-get install -y \ + ftp \ + iputils-ping \ + mc \ + nano \ + net-tools \ + rsync \ && rm -rf /var/lib/apt/lists/* -# Installing requirements w/pip -COPY requirements.txt /tmp -RUN pip install --upgrade pip -RUN pip install --requirement /tmp/requirements.txt - -# Delete root's password so we can do 'su -' -RUN passwd --delete root - -# Install test user and swap to it -RUN groupadd test && useradd -m -g test test -USER test - -# Setting a working directory to home -WORKDIR /home/test - -# Copy a SSH key to the users folder that is used for some test cases -USER root -COPY testing/docker/id_rsa /home/test/.ssh/ -COPY testing/docker/id_rsa.pub /home/test/.ssh/ -RUN chown -R test:test /home/test/.ssh -RUN chmod 400 /home/test/.ssh/id_rsa - -USER test +# Install pyenv +RUN git clone -b "v2.3.0" --single-branch https://github.com/pyenv/pyenv.git /root/.pyenv \ + && eval "$(pyenv init --path)" \ + && eval "$(pyenv init -)" \ +# Install all the Pythons we test + && pyenv install 2.7.18 \ + && pyenv install 3.5.10 \ + && pyenv install 3.6.15 \ + && pyenv install 3.7.13 \ + && pyenv install 3.8.13 \ + && pyenv install 3.9.11 \ + && pyenv install 3.10.3 + +# Tell tox how to find executables, install all requirements +COPY requirements.dev . +COPY requirements.txt . +COPY tox.ini . +RUN python3 -m pip install --upgrade pip setuptools setuptools-scm \ + && python3 -m pip install -rrequirements.dev -rrequirements.txt \ + && python3 -m pip install tox tox-pyenv \ + && pyenv local 2.7.18 3.5.10 3.6.15 3.7.13 3.8.13 3.9.11 3.10.3 + +# set up tiny dummy project to make tox happy +WORKDIR /root +COPY hello-world-master.zip . +RUN unzip -o hello-world-master.zip +COPY requirements.dev hello-world-master +COPY requirements.txt hello-world-master +COPY tox.ini hello-world-master + +# set to dummy workdir +WORKDIR /root/hello-world-master + +# run tox so environss are created as part of image. +RUN tox --notest -- --setup-only + +# cleanup the noise left over +RUN rm -rf /root/.tox/dist/hello_world*.zip \ + rm -rf /root/.tox/distshare/hello_world*.zip \ + rm -rf /root/hello-world-master* \ + rm -rf /root/requirements.* \ + rm -rf /root/tox.ini -# Set final workdir to duplicity -WORKDIR /home/test/duplicity +# Set workdir to duplicity +WORKDIR /root/duplicity Binary files /tmp/tmpws2333ah/ZVRIS9LlZV/duplicity-0.8.22/testing/docker/hello-world-master.zip and /tmp/tmpws2333ah/VDEGHFQN9o/duplicity-0.8.23/testing/docker/hello-world-master.zip differ diff -Nru duplicity-0.8.22/testing/docker/setup.sh duplicity-0.8.23/testing/docker/setup.sh --- duplicity-0.8.22/testing/docker/setup.sh 2022-03-04 18:52:39.000000000 +0000 +++ duplicity-0.8.23/testing/docker/setup.sh 2022-05-15 15:34:02.000000000 +0000 @@ -25,5 +25,4 @@ cd `dirname $0` docker-compose up -d -docker-compose exec -u root duplicity_test python3 setup.py install docker-compose exec duplicity_test /bin/bash diff -Nru duplicity-0.8.22/testing/test_code.py duplicity-0.8.23/testing/test_code.py --- duplicity-0.8.22/testing/test_code.py 2022-03-04 18:52:39.000000000 +0000 +++ duplicity-0.8.23/testing/test_code.py 2022-05-15 15:34:02.000000000 +0000 @@ -53,7 +53,8 @@ for line in output.split(u'\n'): print(line, file=sys.stderr) output = u"" - self.assertTrue(process.returncode in returncodes, output) + self.assertTrue(process.returncode in returncodes, + f"Test failed: returncode = {process.returncode}") @skipCodeTest def test_2to3(self): diff -Nru duplicity-0.8.22/tools/installsnap duplicity-0.8.23/tools/installsnap --- duplicity-0.8.22/tools/installsnap 1970-01-01 00:00:00.000000000 +0000 +++ duplicity-0.8.23/tools/installsnap 2022-05-15 15:34:02.000000000 +0000 @@ -0,0 +1,56 @@ +#!/bin/bash +# -*- Mode:Shell; indent-tabs-mode:nil; tab-width -*- +# +# Copyright 2022 Edgar Soldin , +# Kenneth Loafman +# +# This file is part of duplicity. +# +# Duplicity is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. +# +# Duplicity is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with duplicity; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +set -e + +if [ "`uname`" != "Linux" ]; then + echo "$0 does not run on `uname`" + exit 2 +fi + +if [ "$#" -ne 0 ]; then + echo "usage: $0" + exit 2 +fi + +VERSION=`./setup.py --version` +echo "$0 of ${VERSION} on amd64" + +# only run snap install if not in docker +[ ! -e /.dockerenv ] && { + sudo snap install build/duplicity-${VERSION}/duplicity_${VERSION}_amd64.snap --classic --dangerous + exit $? +} + +# manually "install" duplicity to gain a basic testable version in docker +SNAP_BIN=/snap/bin/duplicity +mkdir -p /snap/duplicity && unsquashfs -d /snap/duplicity/current ./build/duplicity-${VERSION}/duplicity_${VERSION}_amd64.snap + +# create a basic snap runner +cat <"$SNAP_BIN" +#!/bin/sh +export SNAP=/snap/duplicity/current +export PYTHONPATH="\$SNAP/lib/python3.8/site-packages:$HOME/.local/lib/python3.8/site-packages:/lib/python3.8/site-packages" +export PATH="\$SNAP/usr/sbin:\$SNAP/usr/bin:\$SNAP/sbin:\$SNAP/bin:/usr/local/bin:/usr/bin:/bin:/usr/local/games:/usr/games:/snap/bin" +exec "\$SNAP/bin/duplicity" "\$@" +EOF +chmod +x "$SNAP_BIN" diff -Nru duplicity-0.8.22/tools/makechangelog duplicity-0.8.23/tools/makechangelog --- duplicity-0.8.22/tools/makechangelog 2022-03-04 18:52:39.000000000 +0000 +++ duplicity-0.8.23/tools/makechangelog 2022-05-15 15:34:02.000000000 +0000 @@ -1,18 +1,50 @@ #!/bin/bash +# -*- Mode:Shell; indent-tabs-mode:nil; tab-width -*- +# +# Copyright 2022 Kenneth Loafman +# +# This file is part of duplicity. +# +# Duplicity is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. +# +# Duplicity is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with duplicity; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA set -e if [ "$#" -ne 0 ]; then - echo "usage: makechangelog" + echo "usage: $0" exit 2 fi +VERSION=`./setup.py --version` +echo "$0 of ${VERSION}" + gitchangelog > CHANGELOG.md + +# find out if anything has changed +CNT=`git status --porcelain | tee /dev/stderr | wc -l` +if [ $CNT -eq 0 ]; then + echo "*** NOTHING CHANGED, NO COMMITS ***" + exit 0 +fi + +# make sure only CHANGELOG.md has changed CNT=`git status --porcelain | tee /dev/stderr | grep -v " M CHANGELOG.md" | wc -l` if [ $CNT -ne 0 ]; then echo "*** REPO DIRTY, REFUSING TO COMMIT ***" exit 1 fi -git commit -m"Update changelog." -m"[skip ci]" CHANGELOG.md +# finally commit and push +git commit -m"Update changelog." -m"[skip tests]" CHANGELOG.md git push diff -Nru duplicity-0.8.22/tools/makepip duplicity-0.8.23/tools/makepip --- duplicity-0.8.22/tools/makepip 2022-03-04 18:52:39.000000000 +0000 +++ duplicity-0.8.23/tools/makepip 2022-05-15 15:34:02.000000000 +0000 @@ -1,22 +1,41 @@ #!/bin/bash +# -*- Mode:Shell; indent-tabs-mode:nil; tab-width -*- +# +# Copyright 2022 Kenneth Loafman +# +# This file is part of duplicity. +# +# Duplicity is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. +# +# Duplicity is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with duplicity; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA set -e if [ "`uname`" != "Linux" ]; then - echo "makepip does not run on this OS" + echo "$0 does not run on `uname`" exit 2 fi if [ "$#" -ne 0 ]; then - echo "usage: makepip" + echo "usage: $0" exit 2 fi VERSION=`./setup.py --version` -echo "makepip of ${VERSION}" +echo "$0 of ${VERSION}" # make distribution ./setup.py sdist --dist-dir=. # upload to pypi -twine upload duplicity-${VERSION}.tar.gz +twine upload --verbose --non-interactive duplicity-${VERSION}.tar.gz diff -Nru duplicity-0.8.22/tools/makesnap duplicity-0.8.23/tools/makesnap --- duplicity-0.8.22/tools/makesnap 2022-03-04 18:52:39.000000000 +0000 +++ duplicity-0.8.23/tools/makesnap 2022-05-15 15:34:02.000000000 +0000 @@ -1,7 +1,7 @@ #!/bin/bash # -*- Mode:Shell; indent-tabs-mode:nil; tab-width -*- # -# Copyright 2007 Kenneth Loafman +# Copyright 2022 Kenneth Loafman # # This file is part of duplicity. # @@ -22,14 +22,14 @@ set -e if [ "`uname`" != "Linux" ]; then - echo "makesnap does not run on this OS" + echo "$0 does not run on `uname`" exit 2 fi if [ "$#" -gt 1 ]; then - echo "usage: makesnap " + echo "usage: $0 " echo " - default is amd64 only" - echo " - working arch: arm64, amd64" + echo " - working arch: arm64, amd64, armhf, ppc64el" exit 2 elif [ "$#" -eq 1 ]; then BUILD="$1" @@ -38,7 +38,7 @@ fi VERSION=`./setup.py --version` -echo "makesnap of ${VERSION} on ${BUILD}" +echo "$0 of ${VERSION} on ${BUILD}" # make distribution ./setup.py sdist --dist-dir=. @@ -53,9 +53,6 @@ # print version in build copy of snapcraft.yaml head -2 snap/snapcraft.yaml | tail -1 -# make sure we're logged in -snapcraft login --with ~/.snaplogin - if [ ${BUILD} == "amd64" ]; then # build snap locally w/o multipass snapcraft snap --destructive-mode @@ -63,23 +60,3 @@ # build snap via remote build on LP snapcraft remote-build --build-on=${BUILD} --launchpad-accept-public-upload fi - -# push to edge and sign -for f in duplicity_*.snap; do - if [ -e $f ]; then - snapcraft upload $f --release edge - snapcraft sign-build $f --key-name firstprime - fi -done - -# mv into working source tree -if [ -e duplicity_*.snap ]; then - mv duplicity_*.snap ../.. -fi -if [ -e duplicity_*.txt ]; then - mv duplicity_*.txt ../.. -fi - -# clean up -cd ../.. -rm -rf build diff -Nru duplicity-0.8.22/tools/pushsnap duplicity-0.8.23/tools/pushsnap --- duplicity-0.8.22/tools/pushsnap 1970-01-01 00:00:00.000000000 +0000 +++ duplicity-0.8.23/tools/pushsnap 2022-05-15 15:34:02.000000000 +0000 @@ -0,0 +1,53 @@ +#!/bin/bash +# -*- Mode:Shell; indent-tabs-mode:nil; tab-width -*- +# +# Copyright 2022 Kenneth Loafman +# +# This file is part of duplicity. +# +# Duplicity is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. +# +# Duplicity is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with duplicity; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +set -e + +if [ "`uname`" != "Linux" ]; then + echo "$0 does not run on `uname`" + exit 2 +fi + +if [ "$#" -ne 0 ]; then + echo "usage: $0" + exit 2 +fi + +VERSION=`./setup.py --version` +echo "$0 of ${VERSION}" + +# make sure we're logged in +snapcraft login --with ~/.snaplogin + +# push to edge and sign +cd build/duplicity-${VERSION}/ +for f in duplicity_*.snap; do + if [ -e $f ]; then + snapcraft upload $f --release edge + [ ! -e /.dockerenv ] && snapcraft sign-build $f --key-name firstprime + fi +done + +if [ ! -e /.dockerenv ]; then + # mv into working source tree + [ -e duplicity_*.snap ] && mv duplicity_*.snap ../.. + [ -e duplicity_*.txt ] && mv duplicity_*.txt ../.. +fi diff -Nru duplicity-0.8.22/tools/release-prep duplicity-0.8.23/tools/release-prep --- duplicity-0.8.22/tools/release-prep 1970-01-01 00:00:00.000000000 +0000 +++ duplicity-0.8.23/tools/release-prep 2022-05-15 15:34:02.000000000 +0000 @@ -0,0 +1,75 @@ +#!/bin/bash +# -*- Mode:Shell; indent-tabs-mode:nil; tab-width -*- +# +# Copyright 2022 Kenneth Loafman +# +# This file is part of duplicity. +# +# Duplicity is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. +# +# Duplicity is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with duplicity; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +set -e + +if [ "$1" != "" ]; then + REL=$1 +else + echo "usage: $0 version" + exit 1 +fi + +echo "Prepping for release ${REL}" + +while true; do + read -n 1 -p "Is ${REL} the correct version (y/n)?" yn + case $yn in + [Yy]* ) echo; break;; + [Nn]* ) echo; exit;; + * ) echo "Please answer yes or no.";; + esac +done + +if [ `uname` == "Darwin" ]; then + SED=gsed +else + SED=sed +fi + +set -v + +# put in correct version for Launchpad +${SED} -i s/${REL}.dev/${REL}/g setup.py +git commit -a -m"Prep for ${REL}" + +# add release tag and push it +git tag -f rel.${REL} +git push --tags -f origin master -o ci.skip + +# make changelog and move tag to include it +tools/makechangelog +git tag -f rel.${REL} +git push --tags -f origin master + +# update all repos +git push mirror master +git push alpha master -o ci.skip + +# make and sign the release +./setup.py sdist --dist-dir=. +gpg --use-agent -b duplicity-${REL}.tar.gz + +# release tar/sig to Savannah +scp duplicity-${REL}.* loafman@dl.sv.nongnu.org:/releases/duplicity/ + +# move releases to duplicity-releases +mv duplicity-${REL}.* ../duplicity-releases diff -Nru duplicity-0.8.22/tools/release-prep.sh duplicity-0.8.23/tools/release-prep.sh --- duplicity-0.8.22/tools/release-prep.sh 2022-03-04 18:52:39.000000000 +0000 +++ duplicity-0.8.23/tools/release-prep.sh 1970-01-01 00:00:00.000000000 +0000 @@ -1,56 +0,0 @@ -#!/bin/bash - -set -e - -if [ "$1" != "" ]; then - REL=$1 -else - echo "usage: $0 version" - exit 1 -fi - -echo "Prepping for release ${REL}" - -while true; do - read -n 1 -p "Is ${REL} the correct version (y/n)?" yn - case $yn in - [Yy]* ) echo; break;; - [Nn]* ) echo; exit;; - * ) echo "Please answer yes or no.";; - esac -done - -if [ `uname` == "Darwin" ]; then - SED=gsed -else - SED=sed -fi - -set -v - -# put in correct version for Launchpad -${SED} -i s/${REL}.dev/${REL}/g setup.py -git commit -a -m"Prep for ${REL}" - -# add release tag and push it -git tag -f rel.${REL} -git push --tags -f origin master -o ci.skip - -# make changelog and move tag to include it -tools/makechangelog -git tag -f rel.${REL} -git push --tags -f origin master - -# update all repos -git push mirror master -git push alpha master -o ci.skip - -# make and sign the release -./setup.py sdist --dist-dir=. -gpg --use-agent -b duplicity-${REL}.tar.gz - -# release tar/sig to Savannah -scp duplicity-${REL}.* loafman@dl.sv.nongnu.org:/releases/duplicity/ - -# move releases to duplicity-releases -mv duplicity-${REL}.* ../duplicity-releases diff -Nru duplicity-0.8.22/tools/testsnap duplicity-0.8.23/tools/testsnap --- duplicity-0.8.22/tools/testsnap 1970-01-01 00:00:00.000000000 +0000 +++ duplicity-0.8.23/tools/testsnap 2022-05-15 15:34:02.000000000 +0000 @@ -0,0 +1,56 @@ +#!/bin/bash +# -*- Mode:Shell; indent-tabs-mode:nil; tab-width -*- +# +# Copyright 2022 Kenneth Loafman , +# Edgar Soldin +# +# This file is part of duplicity. +# +# Duplicity is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. +# +# Duplicity is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with duplicity; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +set -e + +if [ "`uname`" != "Linux" ]; then + echo "$0 does not run on `uname`" + exit 2 +fi + +if [ "$#" -ne 0 ]; then + echo "usage: $0" + exit 2 +fi + +VERSION=`./setup.py --version` +echo "$0 of ${VERSION}" + +echo -e "\n*** test for the multi-lib snapcraft issue ***" +echo "See: https://bugs.launchpad.net/duplicity/+bug/1965814" +RES=`find /snap/duplicity/current/usr/lib/ -type d -name 'python3.*' | wc -l` +if [ ${RES} -gt 1 ]; then + find /snap/duplicity/current/usr/lib/ -type d -name 'python3.*' + echo -e "WARNING: Multiple python lib versions exist in snap." +else + echo "All Good. Only one version of python libs installed." +fi + +echo -e "\n*** test 'duplicity --version' ***" +/snap/bin/duplicity --version + +echo -e "\n*** test backup ***" +[ -e /tmp/testdup ] && rm -rfv /tmp/testdup +PASSPHRASE=test /snap/bin/duplicity full --name=testsnap ./duplicity file:///tmp/testdup + +echo -e "\n*** test verify ***" +PASSPHRASE=test /snap/bin/duplicity verify --name=testsnap file:///tmp/testdup ./duplicity diff -Nru duplicity-0.8.22/tox.ini duplicity-0.8.23/tox.ini --- duplicity-0.8.22/tox.ini 2022-03-04 18:52:39.000000000 +0000 +++ duplicity-0.8.23/tox.ini 2022-05-15 15:34:02.000000000 +0000 @@ -1,19 +1,44 @@ +# This file is part of duplicity. +# +# Copyright Kenneth Loafman , +# Aaron Whitehouse , +# Edgar Soldin +# +# Duplicity is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. +# +# Duplicity is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with duplicity; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + + [tox] -envlist = py27,py35,py36,py37,py38,py39,py310 +envlist = code,py27,py35,py36,py37,py38,py39,py310 skip_missing_interpreters = false +toxworkdir = {env:TOXWORKDIR:{toxinidir}/.tox} setenv = RUN_CODE_TESTS=0 RUN_COVERAGE=0 [testenv] -deps = -rrequirements.txt +deps = + -rrequirements.txt + -rrequirements.dev usedevelop = True setenv = TOXPYTHON={envpython} passenv = LC_CTYPE LIBRSYNC_DIR PYDEVD + TOXWORKDIR commands = python setup.py build_ext pytest {posargs} @@ -35,7 +60,9 @@ [testenv:coverage] -deps = -rrequirements.txt +deps = + -rrequirements.txt + -rrequirements.dev passenv = LC_CTYPE LIBRSYNC_DIR