1
=== modified file 'src/CMakeLists.txt'
2
--- src/CMakeLists.txt	2016-10-31 07:33:19 +0000
3
+++ src/CMakeLists.txt	2016-11-10 07:22:07 +0000
4
@@ -1,6 +1,7 @@
5
1
1
6
2
add_library(dav-provider-lib STATIC
2
add_library(dav-provider-lib STATIC
7
3
  DavProvider.cpp
3
  DavProvider.cpp
8
4
  DavDownloadJob.cpp
9
4
  DavUploadJob.cpp
5
  DavUploadJob.cpp
10
5
  MultiStatusParser.cpp
6
  MultiStatusParser.cpp
11
6
  item_id.cpp
7
  item_id.cpp
12
7
8
13
=== added file 'src/DavDownloadJob.cpp'
14
--- src/DavDownloadJob.cpp	1970-01-01 00:00:00 +0000
15
+++ src/DavDownloadJob.cpp	2016-11-10 07:22:07 +0000
16
@@ -0,0 +1,183 @@
17
1
#include "DavDownloadJob.h"
18
2
#include "DavProvider.h"
19
3
#include "RetrieveMetadataHandler.h"
20
4
#include "item_id.h"
21
5
22
6
#include <unity/storage/provider/Exceptions.h>
23
7
#include <unity/storage/provider/metadata_keys.h>
24
8
25
9
#include <unistd.h>
26
10
#include <cassert>
27
11
#include <cstdint>
28
12
#include <string>
29
13
30
14
using namespace std;
31
15
using namespace unity::storage::provider;
32
16
using unity::storage::ItemType;
33
17
34
18
namespace
35
19
{
36
20
37
21
string make_download_id()
38
22
{
39
23
    static int counter = 0;
40
24
    return to_string(counter++);
41
25
}
42
26
43
27
constexpr int CHUNK_SIZE = 64 * 1024;
44
28
45
29
}
46
30
47
31
DavDownloadJob::DavDownloadJob(DavProvider const& provider,
48
32
                               string const& item_id,
49
33
                               string const& match_etag,
50
34
                               Context const& ctx)
51
35
    : QObject(), DownloadJob(make_download_id()), provider_(provider),
52
36
      item_id_(item_id)
53
37
{
54
38
    QUrl base_url = provider.base_url(ctx);
55
39
    QNetworkRequest request(id_to_url(item_id, base_url));
56
40
    if (!match_etag.empty())
57
41
    {
58
42
        request.setRawHeader(QByteArrayLiteral("If-Match"),
59
43
                             QByteArray::fromStdString(match_etag));
60
44
    }
61
45
62
46
    reply_ = provider.send_request(
63
47
        request, QByteArrayLiteral("GET"), nullptr, ctx);
64
48
    assert(reply_ != nullptr);
65
49
    reply_->setReadBufferSize(CHUNK_SIZE);
66
50
    connect(reply_, &QNetworkReply::finished,
67
51
            this, &DavDownloadJob::onReplyFinished);
68
52
    connect(reply_, &QIODevice::readyRead,
69
53
            this, &DavDownloadJob::onReplyReadyRead);
70
54
    connect(reply_, &QIODevice::readChannelFinished,
71
55
            this, &DavDownloadJob::onReplyReadChannelFinished);
72
56
    writer_.setSocketDescriptor(
73
57
        dup(write_socket()), QLocalSocket::ConnectedState, QIODevice::WriteOnly);
74
58
    connect(&writer_, &QIODevice::bytesWritten,
75
59
            this, &DavDownloadJob::onSocketBytesWritten);
76
60
}
77
61
78
62
DavDownloadJob::~DavDownloadJob()
79
63
{
80
64
    if (!reply_.isNull())
81
65
    {
82
66
        reply_->deleteLater();
83
67
    }
84
68
}
85
69
86
70
void DavDownloadJob::onReplyFinished()
87
71
{
88
72
    // If we haven't seen HTTP response headers and are in an error
89
73
    // state, report that.
90
74
    if (!seen_header_ && reply_->error() != QNetworkReply::NoError)
91
75
    {
92
76
        handle_error(RemoteCommsException("Error connecting to server: " +
93
77
                                          reply_->errorString().toStdString()));
94
78
    }
95
79
}
96
80
97
81
void DavDownloadJob::onReplyReadyRead()
98
82
{
99
83
    if (error_)
100
84
    {
101
85
        return;
102
86
    }
103
87
104
88
    if (!seen_header_)
105
89
    {
106
90
        seen_header_ = true;
107
91
        auto status = reply_->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
108
92
        if (status != 200)
109
93
        {
110
94
            handle_error(RemoteCommsException("Unexpected status code: " +
111
95
                                              to_string(status)));
112
96
            return;
113
97
        }
114
98
    }
115
99
116
100
    maybe_send_chunk();
117
101
}
118
102
119
103
void DavDownloadJob::onReplyReadChannelFinished()
120
104
{
121
105
    if (error_)
122
106
    {
123
107
        return;
124
108
    }
125
109
126
110
    read_channel_finished_ = true;
127
111
    maybe_send_chunk();
128
112
}
129
113
130
114
void DavDownloadJob::onSocketBytesWritten(int64_t bytes)
131
115
{
132
116
    if (error_)
133
117
    {
134
118
        return;
135
119
    }
136
120
137
121
    bytes_written_ += bytes;
138
122
    maybe_send_chunk();
139
123
}
140
124
141
125
void DavDownloadJob::maybe_send_chunk()
142
126
{
143
127
    assert(bytes_written_ <= bytes_read_);
144
128
    // If there are outstanding writes, do nothing.
145
129
    if (bytes_written_ < bytes_read_)
146
130
    {
147
131
        return;
148
132
    }
149
133
    // If there are no bytes available, return.
150
134
    if (reply_->bytesAvailable() == 0)
151
135
    {
152
136
        // If we've reached the end of the input, and all data has been
153
137
        // written out, signal completion.
154
138
        if (read_channel_finished_ && bytes_written_ == bytes_read_)
155
139
        {
156
140
            writer_.close();
157
141
            report_complete();
158
142
        }
159
143
        return;
160
144
    }
161
145
162
146
    char buffer[CHUNK_SIZE];
163
147
    int n_read = reply_->read(buffer, CHUNK_SIZE);
164
148
    if (n_read < 0)
165
149
    {
166
150
        handle_error(RemoteCommsException("Failed to read from server"));
167
151
        return;
168
152
    }
169
153
    bytes_read_ += n_read;
170
154
171
155
    int n_written = writer_.write(buffer, n_read);
172
156
    if (n_written < 0)
173
157
    {
174
158
        handle_error(ResourceException("Error writing to socket", 0));
175
159
        return;
176
160
    }
177
161
}
178
162
179
163
void DavDownloadJob::handle_error(StorageException const& exc)
180
164
{
181
165
    error_ = true;
182
166
    reply_->close();
183
167
    writer_.close();
184
168
    report_error(std::make_exception_ptr(exc));
185
169
}
186
170
187
171
188
172
boost::future<void> DavDownloadJob::cancel()
189
173
{
190
174
    reply_->abort();
191
175
    writer_.close();
192
176
    return boost::make_ready_future();
193
177
}
194
178
195
179
boost::future<void> DavDownloadJob::finish()
196
180
{
197
181
    return boost::make_exceptional_future<void>(
198
182
        LogicException("finish called before all data sent"));
199
183
}
200
0
184
201
=== added file 'src/DavDownloadJob.h'
202
--- src/DavDownloadJob.h	1970-01-01 00:00:00 +0000
203
+++ src/DavDownloadJob.h	2016-11-10 07:22:07 +0000
204
@@ -0,0 +1,51 @@
205
1
#pragma once
206
2
207
3
#include <QLocalSocket>
208
4
#include <QNetworkReply>
209
5
#include <QObject>
210
6
#include <QPointer>
211
7
#include <QUrl>
212
8
#include <unity/storage/provider/Exceptions.h>
213
9
#include <unity/storage/provider/ProviderBase.h>
214
10
#include <unity/storage/provider/DownloadJob.h>
215
11
216
12
#include <cstdint>
217
13
#include <memory>
218
14
#include <string>
219
15
220
16
class DavProvider;
221
17
222
18
class DavDownloadJob : public QObject, public unity::storage::provider::DownloadJob
223
19
{
224
20
    Q_OBJECT
225
21
public:
226
22
    DavDownloadJob(DavProvider const& provider, std::string const& item_id,
227
23
                   std::string const& match_etag,
228
24
                   unity::storage::provider::Context const& ctx);
229
25
    ~DavDownloadJob();
230
26
231
27
    boost::future<void> cancel() override;
232
28
    boost::future<void> finish() override;
233
29
234
30
private Q_SLOTS:
235
31
    void onReplyFinished();
236
32
    void onReplyReadyRead();
237
33
    void onReplyReadChannelFinished();
238
34
239
35
    void onSocketBytesWritten(int64_t bytes);
240
36
241
37
private:
242
38
    void maybe_send_chunk();
243
39
    void handle_error(unity::storage::provider::StorageException const& exc);
244
40
245
41
    DavProvider const& provider_;
246
42
    std::string const item_id_;
247
43
    QLocalSocket writer_;
248
44
    QPointer<QNetworkReply> reply_;
249
45
250
46
    bool seen_header_ = false;
251
47
    bool error_ = false;
252
48
    bool read_channel_finished_ = false;
253
49
    int64_t bytes_read_ = 0;
254
50
    int64_t bytes_written_ = 0;
255
51
};
256
0
52
257
=== modified file 'src/DavProvider.cpp'
258
--- src/DavProvider.cpp	2016-09-30 12:41:56 +0000
259
+++ src/DavProvider.cpp	2016-11-10 07:22:07 +0000
260
@@ -4,6 +4,7 @@
261
4
#include "ListHandler.h"
4
#include "ListHandler.h"
262
5
#include "LookupHandler.h"
5
#include "LookupHandler.h"
263
6
#include "MetadataHandler.h"
6
#include "MetadataHandler.h"
264
7
#include "DavDownloadJob.h"
265
7
#include "DavUploadJob.h"
8
#include "DavUploadJob.h"
266
8
#include "item_id.h"
9
#include "item_id.h"
267
9
10
268
@@ -87,6 +88,10 @@
269
87
boost::future<unique_ptr<DownloadJob>> DavProvider::download(
88
boost::future<unique_ptr<DownloadJob>> DavProvider::download(
270
88
    string const& item_id, Context const& ctx)
89
    string const& item_id, Context const& ctx)
271
89
{
90
{
272
91
    boost::promise<unique_ptr<DownloadJob>> p;
273
92
    p.set_value(unique_ptr<DownloadJob>(new DavDownloadJob(
274
93
        *this, item_id, string(), ctx)));
275
94
    return p.get_future();
276
90
}
95
}
277
91
96
278
92
boost::future<void> DavProvider::delete_item(
97
boost::future<void> DavProvider::delete_item(
279
93
98
280
=== modified file 'tests/davprovider/davprovider_test.cpp'
281
--- tests/davprovider/davprovider_test.cpp	2016-11-03 08:33:58 +0000
282
+++ tests/davprovider/davprovider_test.cpp	2016-11-10 07:22:07 +0000
283
@@ -12,6 +12,7 @@
284
12
#include <QTemporaryDir>
12
#include <QTemporaryDir>
285
13
#include <QTimer>
13
#include <QTimer>
286
14
#include <unity/storage/qt/client/Account.h>
14
#include <unity/storage/qt/client/Account.h>
287
15
#include <unity/storage/qt/client/Downloader.h>
288
15
#include <unity/storage/qt/client/Exceptions.h>
16
#include <unity/storage/qt/client/Exceptions.h>
289
16
#include <unity/storage/qt/client/Root.h>
17
#include <unity/storage/qt/client/Root.h>
290
17
#include <unity/storage/qt/client/Uploader.h>
18
#include <unity/storage/qt/client/Uploader.h>
291
@@ -45,7 +46,7 @@
292
45
    "nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor "
46
    "nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor "
293
46
    "in reprehenderit in voluptate velit esse cillum dolore eu fugiat "
47
    "in reprehenderit in voluptate velit esse cillum dolore eu fugiat "
294
47
    "nulla pariatur. Excepteur sint occaecat cupidatat non proident, "
48
    "nulla pariatur. Excepteur sint occaecat cupidatat non proident, "
296
48
    "sunt in culpa qui officia deserunt mollit anim id est laborum.";
49
    "sunt in culpa qui officia deserunt mollit anim id est laborum.\n";
297
49
50
298
50
constexpr int SIGNAL_WAIT_TIME = 30000;
51
constexpr int SIGNAL_WAIT_TIME = 30000;
299
51
52
300
@@ -666,6 +667,244 @@
301
666
    }
667
    }
302
667
}
668
}
303
668
669
304
670
TEST_F(DavProviderTests, download)
305
671
{
306
672
    int const segments = 1000;
307
673
    string large_contents;
308
674
    large_contents.reserve(file_contents.size() * segments);
309
675
    for (int i = 0; i < segments; i++)
310
676
    {
311
677
        large_contents += file_contents;
312
678
    }
313
679
    string const full_path = local_file("foo.txt");
314
680
    {
315
681
        int fd = open(full_path.c_str(), O_WRONLY | O_CREAT | O_EXCL, 0644);
316
682
        ASSERT_GT(fd, 0);
317
683
        ASSERT_EQ(large_contents.size(), write(fd, &large_contents[0], large_contents.size())) << strerror(errno);
318
684
        ASSERT_EQ(0, close(fd));
319
685
    }
320
686
321
687
    auto account = get_client();
322
688
    shared_ptr<Root> root;
323
689
    {
324
690
        QFutureWatcher<QVector<shared_ptr<Root>>> watcher;
325
691
        QSignalSpy spy(&watcher, &decltype(watcher)::finished);
326
692
        watcher.setFuture(account->roots());
327
693
        if (spy.count() == 0)
328
694
        {
329
695
            ASSERT_TRUE(spy.wait(SIGNAL_WAIT_TIME));
330
696
        }
331
697
        auto roots = watcher.result();
332
698
        ASSERT_EQ(1, roots.size());
333
699
        root = roots[0];
334
700
    }
335
701
336
702
    shared_ptr<File> file;
337
703
    {
338
704
        QFutureWatcher<shared_ptr<Item>> watcher;
339
705
        QSignalSpy spy(&watcher, &decltype(watcher)::finished);
340
706
        watcher.setFuture(root->get("foo.txt"));
341
707
        if (spy.count() == 0)
342
708
        {
343
709
            ASSERT_TRUE(spy.wait(SIGNAL_WAIT_TIME));
344
710
        }
345
711
        file = dynamic_pointer_cast<File>(watcher.result());
346
712
    }
347
713
    ASSERT_NE(nullptr, file.get());
348
714
349
715
    shared_ptr<Downloader> downloader;
350
716
    {
351
717
        QFutureWatcher<shared_ptr<Downloader>> watcher;
352
718
        QSignalSpy spy(&watcher, &decltype(watcher)::finished);
353
719
        watcher.setFuture(file->create_downloader());
354
720
        if (spy.count() == 0)
355
721
        {
356
722
            ASSERT_TRUE(spy.wait(SIGNAL_WAIT_TIME));
357
723
        }
358
724
        downloader = watcher.result();
359
725
    }
360
726
361
727
    int64_t n_read = 0;
362
728
    auto socket = downloader->socket();
363
729
    QObject::connect(socket.get(), &QIODevice::readyRead,
364
730
                     [socket, &large_contents, &n_read]() {
365
731
                         auto bytes = socket->readAll();
366
732
                         string const expected = large_contents.substr(
367
733
                             n_read, bytes.size());
368
734
                         EXPECT_EQ(expected, bytes.toStdString());
369
735
                         n_read += bytes.size();
370
736
                     });
371
737
    {
372
738
        QSignalSpy spy(socket.get(), &QIODevice::readChannelFinished);
373
739
        ASSERT_TRUE(spy.wait(SIGNAL_WAIT_TIME));
374
740
    }
375
741
376
742
    {
377
743
        QFutureWatcher<void> watcher;
378
744
        QSignalSpy spy(&watcher, &decltype(watcher)::finished);
379
745
        watcher.setFuture(downloader->finish_download());
380
746
        if (spy.count() == 0)
381
747
        {
382
748
            ASSERT_TRUE(spy.wait(SIGNAL_WAIT_TIME));
383
749
        }
384
750
        watcher.waitForFinished(); // to check for errors
385
751
    }
386
752
387
753
    EXPECT_EQ(large_contents.size(), n_read);
388
754
}
389
755
390
756
TEST_F(DavProviderTests, download_short_read)
391
757
{
392
758
    int const segments = 1000;
393
759
    {
394
760
        string full_path = local_file("foo.txt");
395
761
        int fd = open(full_path.c_str(), O_WRONLY | O_CREAT | O_EXCL, 0644);
396
762
        ASSERT_GT(fd, 0);
397
763
        for (int i = 0; i < segments; i++)
398
764
        {
399
765
            ASSERT_EQ(file_contents.size(), write(fd, &file_contents[0], file_contents.size())) << strerror(errno);
400
766
        }
401
767
        ASSERT_EQ(0, close(fd));
402
768
    }
403
769
404
770
    auto account = get_client();
405
771
    shared_ptr<Root> root;
406
772
    {
407
773
        QFutureWatcher<QVector<shared_ptr<Root>>> watcher;
408
774
        QSignalSpy spy(&watcher, &decltype(watcher)::finished);
409
775
        watcher.setFuture(account->roots());
410
776
        if (spy.count() == 0)
411
777
        {
412
778
            ASSERT_TRUE(spy.wait(SIGNAL_WAIT_TIME));
413
779
        }
414
780
        auto roots = watcher.result();
415
781
        ASSERT_EQ(1, roots.size());
416
782
        root = roots[0];
417
783
    }
418
784
419
785
    shared_ptr<File> file;
420
786
    {
421
787
        QFutureWatcher<shared_ptr<Item>> watcher;
422
788
        QSignalSpy spy(&watcher, &decltype(watcher)::finished);
423
789
        watcher.setFuture(root->get("foo.txt"));
424
790
        if (spy.count() == 0)
425
791
        {
426
792
            ASSERT_TRUE(spy.wait(SIGNAL_WAIT_TIME));
427
793
        }
428
794
        file = dynamic_pointer_cast<File>(watcher.result());
429
795
    }
430
796
    ASSERT_NE(nullptr, file.get());
431
797
432
798
    shared_ptr<Downloader> downloader;
433
799
    {
434
800
        QFutureWatcher<shared_ptr<Downloader>> watcher;
435
801
        QSignalSpy spy(&watcher, &decltype(watcher)::finished);
436
802
        watcher.setFuture(file->create_downloader());
437
803
        if (spy.count() == 0)
438
804
        {
439
805
            ASSERT_TRUE(spy.wait(SIGNAL_WAIT_TIME));
440
806
        }
441
807
        downloader = watcher.result();
442
808
    }
443
809
444
810
    {
445
811
        QFutureWatcher<void> watcher;
446
812
        QSignalSpy spy(&watcher, &decltype(watcher)::finished);
447
813
        watcher.setFuture(downloader->finish_download());
448
814
        if (spy.count() == 0)
449
815
        {
450
816
            ASSERT_TRUE(spy.wait(SIGNAL_WAIT_TIME));
451
817
        }
452
818
453
819
        try
454
820
        {
455
821
            watcher.waitForFinished(); // to check for errors
456
822
            FAIL();
457
823
        }
458
824
        catch (LogicException const& e)
459
825
        {
460
826
            EXPECT_EQ("finish called before all data sent", e.error_message());
461
827
        }
462
828
    }
463
829
}
464
830
465
831
TEST_F(DavProviderTests, download_not_found)
466
832
{
467
833
    auto account = get_client();
468
834
    make_file("foo.txt");
469
835
470
836
    shared_ptr<Root> root;
471
837
    {
472
838
        QFutureWatcher<QVector<shared_ptr<Root>>> watcher;
473
839
        QSignalSpy spy(&watcher, &decltype(watcher)::finished);
474
840
        watcher.setFuture(account->roots());
475
841
        if (spy.count() == 0)
476
842
        {
477
843
            ASSERT_TRUE(spy.wait(SIGNAL_WAIT_TIME));
478
844
        }
479
845
        auto roots = watcher.result();
480
846
        ASSERT_EQ(1, roots.size());
481
847
        root = roots[0];
482
848
    }
483
849
484
850
    shared_ptr<File> file;
485
851
    {
486
852
        QFutureWatcher<shared_ptr<Item>> watcher;
487
853
        QSignalSpy spy(&watcher, &decltype(watcher)::finished);
488
854
        watcher.setFuture(root->get("foo.txt"));
489
855
        if (spy.count() == 0)
490
856
        {
491
857
            ASSERT_TRUE(spy.wait(SIGNAL_WAIT_TIME));
492
858
        }
493
859
        file = dynamic_pointer_cast<File>(watcher.result());
494
860
    }
495
861
    ASSERT_NE(nullptr, file.get());
496
862
497
863
    ASSERT_EQ(0, unlink(local_file("foo.txt").c_str()));
498
864
499
865
    shared_ptr<Downloader> downloader;
500
866
    {
501
867
        QFutureWatcher<shared_ptr<Downloader>> watcher;
502
868
        QSignalSpy spy(&watcher, &decltype(watcher)::finished);
503
869
        watcher.setFuture(file->create_downloader());
504
870
        if (spy.count() == 0)
505
871
        {
506
872
            ASSERT_TRUE(spy.wait(SIGNAL_WAIT_TIME));
507
873
        }
508
874
        downloader = watcher.result();
509
875
    }
510
876
511
877
    auto socket = downloader->socket();
512
878
    QObject::connect(socket.get(), &QIODevice::readyRead,
513
879
                     [socket]() {
514
880
                         socket->readAll();
515
881
                     });
516
882
    {
517
883
        QSignalSpy spy(socket.get(), &QIODevice::readChannelFinished);
518
884
        ASSERT_TRUE(spy.wait(SIGNAL_WAIT_TIME));
519
885
    }
520
886
521
887
    {
522
888
        QFutureWatcher<void> watcher;
523
889
        QSignalSpy spy(&watcher, &decltype(watcher)::finished);
524
890
        watcher.setFuture(downloader->finish_download());
525
891
        if (spy.count() == 0)
526
892
        {
527
893
            ASSERT_TRUE(spy.wait(SIGNAL_WAIT_TIME));
528
894
        }
529
895
530
896
        try
531
897
        {
532
898
            watcher.waitForFinished(); // to check for errors
533
899
            FAIL();
534
900
        }
535
901
        catch (RemoteCommsException const& e)
536
902
        {
537
903
            EXPECT_EQ("Unexpected status code: 404", e.error_message());
538
904
        }
539
905
    }
540
906
}
541
907
542
669
int main(int argc, char**argv)
908
int main(int argc, char**argv)
543
670
{
909
{
544
671
    QCoreApplication app(argc, argv);
910
    QCoreApplication app(argc, argv);
Status:	Merged
Approved by:	James Henstridge on 2016-11-10
Approved revision:	22
Merged at revision:	14
Proposed branch:	lp:~jamesh/storage-provider-webdav/download-job
Merge into:	lp:storage-provider-webdav/devel
Diff against target:	544 lines (+480/-1) 5 files modified src/CMakeLists.txt (+1/-0) src/DavDownloadJob.cpp (+183/-0) src/DavDownloadJob.h (+51/-0) src/DavProvider.cpp (+5/-0) tests/davprovider/davprovider_test.cpp (+240/-1)
To merge this branch:	bzr merge lp:~jamesh/storage-provider-webdav/download-job
Related bugs:	Link a bug report
Reviewer	Review Type	Date Requested	Status
unity-api-1-bot	continuous-integration		Approve on 2016-11-10
Michi Henning (community)		2016-11-08	Approve on 2016-11-10
Review via email: mp+310275@code.launchpad.net