Merge lp:~zorba-coders/zorba/feature-ftp-client into lp:zorba

Proposed by Paul J. Lucas
Status: Merged
Approved by: Paul J. Lucas
Approved revision: 11718
Merged at revision: 11686
Proposed branch: lp:~zorba-coders/zorba/feature-ftp-client
Merge into: lp:zorba
Diff against target: 4590 lines (+3429/-694)
41 files modified
CMakeCPack.cmake (+1/-1)
CMakeLists.txt (+0/-1)
NOTICE.txt (+10/-0)
NOTICE.xml (+11/-0)
include/zorba/util/mem_streambuf.h (+8/-4)
modules/CMakeLists.txt (+2/-0)
modules/ftp-client/CMakeLists.txt (+26/-0)
modules/ftp-client/ftp-client.xq (+521/-0)
modules/ftp-client/ftp-client.xq.src/ftp_connections.cpp (+67/-0)
modules/ftp-client/ftp-client.xq.src/ftp_connections.h (+87/-0)
modules/ftp-client/ftp-client.xq.src/ftp_functions.cpp (+934/-0)
modules/ftp-client/ftp-client.xq.src/ftp_functions.h (+159/-0)
modules/ftp-client/ftp-client.xq.src/ftp_module.cpp (+101/-0)
modules/ftp-client/ftp-client.xq.src/ftp_module.h (+56/-0)
modules/ftp-client/ftp-client.xq.src/ftpparse.c (+448/-0)
modules/ftp-client/ftp-client.xq.src/ftpparse.h (+51/-0)
modules/http-client/CMakeLists.txt (+21/-33)
modules/http-client/json/http-client.xq (+1/-0)
modules/http-client/json/http-client.xq.src/curl_stream_buffer.cpp (+0/-379)
modules/http-client/json/http-client.xq.src/curl_stream_buffer.h (+0/-193)
modules/http-client/json/http-client.xq.src/http_client.cpp (+1/-1)
modules/http-client/json/http-client.xq.src/http_response_parser.cpp (+9/-12)
modules/http-client/json/http-client.xq.src/http_response_parser.h (+4/-10)
modules/http-client/json/http-client.xq.src/inform_data_read.cpp (+0/-23)
modules/http-client/json/http-client.xq.src/inform_data_read.h (+0/-28)
modules/util-curl/CMakeLists.txt (+50/-0)
modules/util-curl/ZorbaUtilCurlModuleConfig.cmake (+19/-0)
modules/util-curl/include/util/curl_streambuf.h (+316/-0)
modules/util-curl/src/CMakeLists.txt (+36/-0)
modules/util-curl/src/curl_streambuf.cpp (+389/-0)
modules/util-curl/src/util-curl.cpp (+59/-0)
modules/util-curl/src/util-curl.xq (+34/-0)
src/api/CMakeLists.txt (+1/-0)
src/api/base64_util.cpp (+1/-1)
src/api/hexbinary_util.cpp (+1/-1)
src/api/mem_streambuf.cpp (+1/-2)
src/runtime/csv/pregenerated/csv.h (+1/-1)
src/runtime/json/json_impl.cpp (+1/-1)
src/runtime/json/snelson.cpp (+1/-1)
src/runtime/spec/csv/csv.xml (+1/-1)
src/util/CMakeLists.txt (+0/-1)
To merge this branch: bzr merge lp:~zorba-coders/zorba/feature-ftp-client
Reviewer Review Type Date Requested Status
Matthias Brantner Approve
Paul J. Lucas Approve
Review via email: mp+201082@code.launchpad.net

Commit message

1. New FTP client.
2. Moved curl_streambuf code to shared util-curl module.
3. Made existing http client use shared curl_streambuf.

Description of the change

1. New FTP client.
2. Moved curl_streambuf code to shared util-curl module.
3. Made existing http client use shared curl_streambuf.

To post a comment you must log in.
11716. By Paul J. Lucas

Fixes.

11717. By Paul J. Lucas

Merge from trunk.

11718. By Paul J. Lucas

Added mention of ftpparse in NOTICE.

Revision history for this message
Paul J. Lucas (paul-lucas) :
review: Approve
11719. By Paul J. Lucas

Moved curl_streambuf.h into util subdirectory.

11720. By Paul J. Lucas

Added #if tests for cURL options.

11721. By Paul J. Lucas

Added ZORBA_DLL_PUBLIC.

11722. By Paul J. Lucas

Added ZORBA_DLL_PUBLIC.

11723. By Paul J. Lucas

Added ZORBA_DLL_PUBLIC.

Revision history for this message
Matthias Brantner (matthias-brantner) wrote :

- I tried
 import module namespace ftp = "http://zorba.io/modules/ftp-client";
 ftp:list(ftp:connect("ftp://192.168.1.18", {"user": "brantner", "password" : "XXX"}), "/")

 doesn't list anything on the default os x ftp server

- Nothing seems to happen if I provide a wrong password.

- Remove the disconnect function.
- Can you make the connect, get-*, and list functions nonsequential (i.e. deterministic)
- Also add new functions signatures connect-nondeterministc, get-*-nondeterministic, and list-nondeterministic. The implementation for those functions can be the same but they should be annotated as nondeterministic.

review: Needs Fixing
Revision history for this message
Matthias Brantner (matthias-brantner) wrote :

declare function
ftp:connect( $uri as string, $options as object )
  as anyURI external;

declare %an:nondeterministic function
ftp:connect-nondeterministic( $uri as string, $options as object )
  as anyURI external;

declare %an:sequential function
ftp:delete( $conn as string, $remote-path as string )
  external;

declare function
ftp:get-binary( $conn as anyURI, $remote-path as string )
  as base64Binary external;

declare %an:nondeterministic function
ftp:get-binary-nondeterministic( $conn as anyURI, $remote-path as string )
  as base64Binary external;

declare function
ftp:get-text( $conn as anyURI, $remote-path as string, $encoding as string )
  as string external;

declare %an:nondeterministic function
ftp:get-text-nondeterministic( $conn as anyURI, $remote-path as string, $encoding as string )
  as string external;

declare function
ftp:get-text( $conn as anyURI, $remote-path as string )
  as string
{
  ftp:get-text( $conn, $remote-path, "UTF-8" )
};

declare %an:nondeterministic function
ftp:get-text-nondeterministic( $conn as anyURI, $remote-path as string )
  as string
{
  ftp:get-text( $conn, $remote-path, "UTF-8" )
};

declare function
ftp:list( $conn as anyURI, $remote-path as string )
  as object* external;

declare %an:nondeterministic function
ftp:list-nondeterministic( $conn as anyURI, $remote-path as string )
  as object* external;

declare %an:sequential function
ftp:mkdir( $conn as string, $remote-path as string )
  external;

declare %an:sequential function
ftp:put-binary( $conn as anyURI, $binary as base64Binary,
                $remote-path as string )
  external;

declare %an:sequential function
ftp:put-text( $conn as anyURI, $text as string, $remote-path as string,
              $encoding as string )
  external;

declare %an:sequential function
ftp:put-text( $conn as anyURI, $text as string, $remote-path as string )
{
  ftp:put-text( $conn, $text, $remote-path, "UTF-8" )
};

declare %an:sequential function
ftp:rename( $conn as string, $remote-from-path as string,
            $remote-to-path as string )
  external;

declare %an:sequential function
ftp:rmdir( $conn as string, $remote-path as string )
  external;

11724. By Paul J. Lucas

Now setting verbose_ to false by default.

11725. By Paul J. Lucas

Comment fix.

11726. By Paul J. Lucas

Wanted changes.

11727. By Paul J. Lucas

Set stuff to 0 to eliminate warnings.

Revision history for this message
Matthias Brantner (matthias-brantner) :
review: Approve
Revision history for this message
Zorba Build Bot (zorba-buildbot) wrote :

Validation queue starting for the following merge proposals:
https://code.launchpad.net/~zorba-coders/zorba/feature-ftp-client/+merge/201082

Progress dashboard at http://jenkins.lambda.nu/view/ValidationQueue

Revision history for this message
Zorba Build Bot (zorba-buildbot) wrote :

Validation queue result for https://code.launchpad.net/~zorba-coders/zorba/feature-ftp-client/+merge/201082

Stage "TestZorbaUbuntu" failed.
131 tests failed (8754 total tests run).

Check test results at http://jenkins.lambda.nu:8180/job/TestZorbaUbuntu/432/testReport/ to view the results.

11728. By Paul J. Lucas

Removed call to curl_easy_reset().

11729. By Paul J. Lucas

More comments.

Revision history for this message
Zorba Build Bot (zorba-buildbot) wrote :

Validation queue starting for the following merge proposals:
https://code.launchpad.net/~zorba-coders/zorba/feature-ftp-client/+merge/201082

Progress dashboard at http://jenkins.lambda.nu/view/ValidationQueue

Revision history for this message
Zorba Build Bot (zorba-buildbot) wrote :

Validation queue result for https://code.launchpad.net/~zorba-coders/zorba/feature-ftp-client/+merge/201082

Stage "TestZorbaUbuntu" failed.
1 tests failed (8754 total tests run).

Check test results at http://jenkins.lambda.nu:8180/job/TestZorbaUbuntu/433/testReport/ to view the results.

11730. By Paul J. Lucas

Updated documentation to return () explicitly.

Revision history for this message
Zorba Build Bot (zorba-buildbot) wrote :

Validation queue starting for the following merge proposals:
https://code.launchpad.net/~zorba-coders/zorba/feature-ftp-client/+merge/201082

Progress dashboard at http://jenkins.lambda.nu/view/ValidationQueue

Revision history for this message
Zorba Build Bot (zorba-buildbot) wrote :

Validation queue succeeded - proposal merged!

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'CMakeCPack.cmake'
2--- CMakeCPack.cmake 2013-08-06 16:51:59 +0000
3+++ CMakeCPack.cmake 2014-01-13 19:25:59 +0000
4@@ -128,7 +128,7 @@
5 SET(CPACK_POSTFLIGHT_SCRIPT "${CMAKE_BINARY_DIR}/osx_postflight.sh")
6 CONFIGURE_FILE("${CMAKE_SOURCE_DIR}/scripts/osx_postflight.sh.in"
7 "${CMAKE_BINARY_DIR}/osx_postflight.sh")
8- MESSAGE ( STATUS "script = "${CPACK_POSTFLIGHT_SCRIPT} )
9+ MESSAGE ( STATUS "script = ${CPACK_POSTFLIGHT_SCRIPT}" )
10 ENDIF ( APPLE )
11 INCLUDE(CPack)
12 INCLUDE(CPack.cmake)
13
14=== modified file 'CMakeLists.txt'
15--- CMakeLists.txt 2013-11-04 22:29:39 +0000
16+++ CMakeLists.txt 2014-01-13 19:25:59 +0000
17@@ -591,7 +591,6 @@
18 # non-core modules are available.
19
20 ADD_SUBDIRECTORY(bin)
21-
22 ADD_SUBDIRECTORY(test)
23 ADD_SUBDIRECTORY(config)
24 ADD_SUBDIRECTORY(schemas)
25
26=== modified file 'NOTICE.txt'
27--- NOTICE.txt 2013-10-30 15:22:48 +0000
28+++ NOTICE.txt 2014-01-13 19:25:59 +0000
29@@ -474,6 +474,16 @@
30
31 ========================================================================
32
33+----------------------------------------------------
34+
35+modules/ftp-client/ftp-client.xq.src/ftpparse.h
36+modules/ftp-client/ftp-client.xq.src/ftpparse.c
37+
38+Copyright: 2000 D. J. Bernstein
39+Website: http://cr.yp.to/ftpparse.html
40+
41+ Commercial use is fine, if you let me know what programs you're using this in.
42+
43
44 External libraries used by this project:
45 ----------------------------------------------------
46
47=== modified file 'NOTICE.xml'
48--- NOTICE.xml 2013-09-18 18:51:34 +0000
49+++ NOTICE.xml 2014-01-13 19:25:59 +0000
50@@ -438,6 +438,16 @@
51 ========================================================================
52 </foreign-notice>
53 </foreign-files>
54+ <foreign-files>
55+ <file>modules/ftp-client/ftp-client.xq.src/ftpparse.h</file>
56+ <file>modules/ftp-client/ftp-client.xq.src/ftpparse.c</file>
57+ <copyright>2000 D. J. Bernstein</copyright>
58+ <website>http://cr.yp.to/ftpparse.html</website>
59+ <foreign-notice>
60+ Commercial use is fine, if you let me know what programs you're
61+ using this in.
62+ </foreign-notice>
63+ </foreign-files>
64
65 <external-lib mandatory="true">
66 <name>LIBXML2</name>
67@@ -699,3 +709,4 @@
68 </foreign-notice>
69 </external-app>
70 </notice>
71+<!-- vim:set et sw=2 ts=2: -->
72
73=== renamed file 'src/util/mem_streambuf.h' => 'include/zorba/util/mem_streambuf.h'
74--- src/util/mem_streambuf.h 2013-08-01 00:31:20 +0000
75+++ include/zorba/util/mem_streambuf.h 2014-01-13 19:25:59 +0000
76@@ -14,11 +14,15 @@
77 * limitations under the License.
78 */
79
80-#ifndef ZORBA_MMAP_STREAMBUF_H
81-#define ZORBA_MMAP_STREAMBUF_H
82+#ifndef ZORBA_API_MEM_STREAMBUF_H
83+#define ZORBA_API_MEM_STREAMBUF_H
84
85+// standard
86 #include <streambuf>
87
88+// Zorba
89+#include <zorba/config.h>
90+
91 namespace zorba {
92
93 ///////////////////////////////////////////////////////////////////////////////
94@@ -26,7 +30,7 @@
95 /**
96 * A %mem_streambuf is-a std::streambuf for a fixed-size chunk of memory.
97 */
98-class mem_streambuf : public std::streambuf {
99+class ZORBA_DLL_PUBLIC mem_streambuf : public std::streambuf {
100 public:
101 typedef std::streambuf::char_type char_type;
102 typedef std::streambuf::int_type int_type;
103@@ -118,5 +122,5 @@
104 ///////////////////////////////////////////////////////////////////////////////
105
106 } // namespace zorba
107-#endif /* ZORBA_MMAP_STREAMBUF_H */
108+#endif /* ZORBA_API_MEM_STREAMBUF_H */
109 /* vim:set et sw=2 ts=2: */
110
111=== modified file 'modules/CMakeLists.txt'
112--- modules/CMakeLists.txt 2013-10-16 22:04:45 +0000
113+++ modules/CMakeLists.txt 2014-01-13 19:25:59 +0000
114@@ -12,11 +12,13 @@
115 # See the License for the specific language governing permissions and
116 # limitations under the License.
117
118+ADD_SUBDIRECTORY(util-curl)
119 ADD_SUBDIRECTORY(atomic)
120 ADD_SUBDIRECTORY(com)
121 ADD_SUBDIRECTORY(full-text)
122 ADD_SUBDIRECTORY(functx)
123 ADD_SUBDIRECTORY(dateTime)
124+ADD_SUBDIRECTORY(ftp-client)
125 ADD_SUBDIRECTORY(http-client)
126 ADD_SUBDIRECTORY(item)
127 ADD_SUBDIRECTORY(json)
128
129=== added directory 'modules/ftp-client'
130=== added file 'modules/ftp-client/CMakeLists.txt'
131--- modules/ftp-client/CMakeLists.txt 1970-01-01 00:00:00 +0000
132+++ modules/ftp-client/CMakeLists.txt 2014-01-13 19:25:59 +0000
133@@ -0,0 +1,26 @@
134+# Copyright 2006-2013 The FLWOR Foundation.
135+#
136+# Licensed under the Apache License, Version 2.0 (the "License");
137+# you may not use this file except in compliance with the License.
138+# You may obtain a copy of the License at
139+#
140+# http://www.apache.org/licenses/LICENSE-2.0
141+#
142+# Unless required by applicable law or agreed to in writing, software
143+# distributed under the License is distributed on an "AS IS" BASIS,
144+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
145+# See the License for the specific language governing permissions and
146+# limitations under the License.
147+
148+IF (ZORBA_HAVE_CURL)
149+ SET(ZorbaUtilCurlModule_DIR "../util-curl")
150+ FIND_PACKAGE(ZorbaUtilCurlModule REQUIRED)
151+ IF (ZorbaUtilCurlModule_FOUND)
152+ INCLUDE_DIRECTORIES("${ZorbaUtilCurlModule_INCLUDE_DIRS}")
153+ DECLARE_ZORBA_MODULE( FILE ftp-client.xq VERSION 1.0
154+ URI "http://zorba.io/modules/ftp-client"
155+ LINK_LIBRARIES ${CURL_LIBRARIES} ${ZorbaUtilCurlModule_LIBS})
156+ ENDIF (ZorbaUtilCurlModule_FOUND)
157+ENDIF (ZORBA_HAVE_CURL)
158+
159+# vim:set et sw=2 ts=2:
160
161=== added file 'modules/ftp-client/ftp-client.xq'
162--- modules/ftp-client/ftp-client.xq 1970-01-01 00:00:00 +0000
163+++ modules/ftp-client/ftp-client.xq 2014-01-13 19:25:59 +0000
164@@ -0,0 +1,521 @@
165+jsoniq version "1.0";
166+
167+(:
168+ : Copyright 2006-2013 The FLWOR Foundation.
169+ :
170+ : Licensed under the Apache License, Version 2.0 (the "License");
171+ : you may not use this file except in compliance with the License.
172+ : You may obtain a copy of the License at
173+ :
174+ : http://www.apache.org/licenses/LICENSE-2.0
175+ :
176+ : Unless required by applicable law or agreed to in writing, software
177+ : distributed under the License is distributed on an "AS IS" BASIS,
178+ : WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
179+ : See the License for the specific language governing permissions and
180+ : limitations under the License.
181+ :)
182+
183+(:===========================================================================:)
184+
185+(:~
186+ : This module provides functions for performing FTP commands.
187+ : <p/>
188+ : @author Paul J. Lucas
189+ : @library <a href="http://curl.haxx.se/">cURL Library</a>
190+ : @project Zorba/Input Output/FTP Client
191+ :)
192+module namespace ftp = "http://zorba.io/modules/ftp-client";
193+
194+declare namespace an = "http://zorba.io/annotations";
195+declare namespace ver = "http://zorba.io/options/versioning";
196+
197+declare option ver:module-version "1.0";
198+
199+(:~
200+ : Attempts to connect to an FTP server specified by the given URI.
201+ :
202+ : @param $uri The address of the FTP server to connect to.
203+ : It may either be simple host-name
204+ : (<code>ftp.example.com</code>)
205+ : or a URI using the <code>ftp</code>
206+ : or <code>ftps</code> (FTP over SSL/TLS)
207+ : schemes
208+ : (<code>ftp://ftp.example.com</code>).
209+ : If the latter,
210+ : the URI may also contain
211+ : <code>username</code>,
212+ : <code>password</code>,
213+ : and
214+ : <code>port</code>
215+ : authority subcomponents
216+ : per RFC 3986.
217+ : @param $options The options to use:
218+ : <dl>
219+ : <dt><code>user</code></dt>
220+ : <dd>
221+ : The user to log in as;
222+ : default: <code>"ftp"</code> (for anonymous FTP).
223+ : If <code>$uri</code> is actually a URI instead of a simple host-name,
224+ : the user, if specified, must be part of the URI
225+ : and the value of this option is ignored.
226+ : </dd>
227+ : <dt><code>password</code></dt>
228+ : <dd>
229+ : The password to use to log in;
230+ : default: <code>"ftp@example.com"</code> (for anonymous FTP).
231+ : If <code>$uri</code> is actually a URI instead of a simple host-name,
232+ : the password, if specified, must be part of the URI
233+ : and the value of this option is ignored.
234+ : </dd>
235+ : <dt><code>port</code></dt>
236+ : <dd>
237+ : The port number to use;
238+ : default: whatever the default for the protocol is.
239+ : If <code>$uri</code> is actually a URI instead of a simple host-name,
240+ : the port, if specified, must be part of the URI
241+ : and the value of this option is ignored.
242+ : </dd>
243+ : <dt><code>protocol</code></dt>
244+ : <dd>
245+ : The protocol to use,
246+ : either <code>"ftp'</code> or <code>"ftps"</code>;
247+ : default: <code>"ftp"</code>.
248+ : If <code>$uri</code> is actually a URI instead of a simple host-name,
249+ : the protocol, if specified, must be part of the URI
250+ : and the value of this option is ignored.
251+ : </dd>
252+ : <dt><code>SSL-communication</code></dt>
253+ : <dd>
254+ : Whether to use SSL/TLS, one of:
255+ : <dl>
256+ : <dt><code>"none"</code></dt>
257+ : <dd>Don't use SSL.</dd>
258+ : <dt><code>"try"</code></dt>
259+ : <dd>Try using SSL, but, if unsuccessful, continue anyway.</dd>
260+ : <dt><code>"control"</code></dt>
261+ : <dd>Require SSL for the control connection.</dd>
262+ : <dt><code>"all"</code></dt>
263+ : <dd>Require SSL for all communication.</dd>
264+ : </dl>
265+ : default:
266+ : <code>"none"</code> for protocol <code>"ftp"</code>
267+ : and
268+ : <code>"all"</code> for protocol <code>"ftps"</code>.
269+ : Note that any value other than <code>"none"</code>
270+ : for <code>"ftp"</code> implies <em>explicit</em> SSL;
271+ : use of <code>"ftps"</code> implies <em>implicit</em> SSL.
272+ : See "<a href="http://en.wikipedia.org/wiki/FTPS#Methods_of_invoking_security">Methods of invoking security</a>."
273+ : </dd>
274+ : <dt><code>SSL-verify</code></dt>
275+ : <dd>
276+ : When doing FTP over SSL/TLS,
277+ : whether to verify the authenticity of the server's certificate
278+ : and that the certificate is for that server;
279+ : default: <code>true</code>.
280+ : (You should <em>never</em> set this to <code>false</code>
281+ : unless you are testing your own FTP server
282+ : with a self-signed certificate.)
283+ : </dd>
284+ : <dt><code>trace</code></dt>
285+ : <dd>
286+ : Whether to emit information to standard error
287+ : tracing the communication between the FTP client and server;
288+ : default: <code>false</code>.
289+ : </dd>
290+ : </dl>
291+ : @return an opaque URI that serves as a connection handle to be used with
292+ : other functions in this module.
293+ : @error ftp:ALREADY_CONNECTED if <code>$uri</code> is already connected to.
294+ : @error ftp:INVALID_ARGUMENT if any option is invalid.
295+ : @error ftp:FTP_ERROR if there was some other FTP error.
296+ :)
297+declare function
298+ftp:connect( $uri as string, $options as object )
299+ as anyURI external;
300+
301+(:~
302+ : Attempts to connect to an FTP server specified by the given URI.
303+ :
304+ : @param $uri The address of the FTP server to connect to.
305+ : It may either be simple host-name
306+ : (<code>ftp.example.com</code>)
307+ : or a URI using the <code>ftp</code>
308+ : or <code>ftps</code> (FTP over SSL/TLS)
309+ : schemes
310+ : (<code>ftp://ftp.example.com</code>).
311+ : If the latter,
312+ : the URI may also contain
313+ : <code>username</code>,
314+ : <code>password</code>,
315+ : and
316+ : <code>port</code>
317+ : authority subcomponents
318+ : per RFC 3986.
319+ : @param $options The options to use:
320+ : <dl>
321+ : <dt><code>user</code></dt>
322+ : <dd>
323+ : The user to log in as;
324+ : default: <code>"ftp"</code> (for anonymous FTP).
325+ : If <code>$uri</code> is actually a URI instead of a simple host-name,
326+ : the user, if specified, must be part of the URI
327+ : and the value of this option is ignored.
328+ : </dd>
329+ : <dt><code>password</code></dt>
330+ : <dd>
331+ : The password to use to log in;
332+ : default: <code>"ftp@example.com"</code> (for anonymous FTP).
333+ : If <code>$uri</code> is actually a URI instead of a simple host-name,
334+ : the password, if specified, must be part of the URI
335+ : and the value of this option is ignored.
336+ : </dd>
337+ : <dt><code>port</code></dt>
338+ : <dd>
339+ : The port number to use;
340+ : default: whatever the default for the protocol is.
341+ : If <code>$uri</code> is actually a URI instead of a simple host-name,
342+ : the port, if specified, must be part of the URI
343+ : and the value of this option is ignored.
344+ : </dd>
345+ : <dt><code>protocol</code></dt>
346+ : <dd>
347+ : The protocol to use,
348+ : either <code>"ftp'</code> or <code>"ftps"</code>;
349+ : default: <code>"ftp"</code>.
350+ : If <code>$uri</code> is actually a URI instead of a simple host-name,
351+ : the protocol, if specified, must be part of the URI
352+ : and the value of this option is ignored.
353+ : </dd>
354+ : <dt><code>SSL-communication</code></dt>
355+ : <dd>
356+ : Whether to use SSL/TLS, one of:
357+ : <dl>
358+ : <dt><code>"none"</code></dt>
359+ : <dd>Don't use SSL.</dd>
360+ : <dt><code>"try"</code></dt>
361+ : <dd>Try using SSL, but, if unsuccessful, continue anyway.</dd>
362+ : <dt><code>"control"</code></dt>
363+ : <dd>Require SSL for the control connection.</dd>
364+ : <dt><code>"all"</code></dt>
365+ : <dd>Require SSL for all communication.</dd>
366+ : </dl>
367+ : default:
368+ : <code>"none"</code> for protocol <code>"ftp"</code>
369+ : and
370+ : <code>"all"</code> for protocol <code>"ftps"</code>.
371+ : Note that any value other than <code>"none"</code>
372+ : for <code>"ftp"</code> implies <em>explicit</em> SSL;
373+ : use of <code>"ftps"</code> implies <em>implicit</em> SSL.
374+ : See "<a href="http://en.wikipedia.org/wiki/FTPS#Methods_of_invoking_security">Methods of invoking security</a>."
375+ : </dd>
376+ : <dt><code>SSL-verify</code></dt>
377+ : <dd>
378+ : When doing FTP over SSL/TLS,
379+ : whether to verify the authenticity of the server's certificate
380+ : and that the certificate is for that server;
381+ : default: <code>true</code>.
382+ : (You should <em>never</em> set this to <code>false</code>
383+ : unless you are testing your own FTP server
384+ : with a self-signed certificate.)
385+ : </dd>
386+ : <dt><code>trace</code></dt>
387+ : <dd>
388+ : Whether to emit information to standard error
389+ : tracing the communication between the FTP client and server;
390+ : default: <code>false</code>.
391+ : </dd>
392+ : </dl>
393+ : @return an opaque URI that serves as a connection handle to be used with
394+ : other functions in this module.
395+ : @error ftp:ALREADY_CONNECTED if <code>$uri</code> is already connected to.
396+ : @error ftp:INVALID_ARGUMENT if any option is invalid.
397+ : @error ftp:FTP_ERROR if there was some other FTP error.
398+ :)
399+declare %an:nondeterministic function
400+ftp:connect-nondeterministic( $uri as string, $options as object )
401+ as anyURI
402+{
403+ ftp:connect( $uri, $options )
404+};
405+
406+(:~
407+ : Deletes a file from the FTP server.
408+ :
409+ : @param $conn The opaque URI connection handle previously returned by
410+ : <code>ftp:connect()</code>.
411+ : @param $remote-path The path of the file to delete.
412+ : @return the empty sequence.
413+ : @error ftp:INVALID_ARGUMENT if <code>$remote-path</code> is empty.
414+ : @error ftp:NOT_CONNECTED if <code>$conn</code> is either an invalid handle
415+ : or is no longer a valid handle.
416+ : @error ftp:FTP_ERROR if there was some other FTP error.
417+ :)
418+declare %an:sequential function
419+ftp:delete( $conn as string, $remote-path as string )
420+ as () external;
421+
422+(:~
423+ : Gets a binary file from the FTP server.
424+ :
425+ : @param $conn The opaque URI connection handle previously returned by
426+ : <code>ftp:connect()</code>.
427+ : @param $remote-path The path of the file to get.
428+ : @return the binary content of <code>$remote-path</code>.
429+ : @error ftp:INVALID_ARGUMENT if <code>$remote-path</code> is empty.
430+ : @error ftp:NOT_CONNECTED if <code>$conn</code> is either an invalid handle
431+ : or is no longer a valid handle.
432+ : @error ftp:FTP_ERROR if there was some other FTP error.
433+ :)
434+declare function
435+ftp:get-binary( $conn as anyURI, $remote-path as string )
436+ as base64Binary external;
437+
438+(:~
439+ : Gets a binary file from the FTP server.
440+ :
441+ : @param $conn The opaque URI connection handle previously returned by
442+ : <code>ftp:connect()</code>.
443+ : @param $remote-path The path of the file to get.
444+ : @return the binary content of <code>$remote-path</code>.
445+ : @error ftp:INVALID_ARGUMENT if <code>$remote-path</code> is empty.
446+ : @error ftp:NOT_CONNECTED if <code>$conn</code> is either an invalid handle
447+ : or is no longer a valid handle.
448+ : @error ftp:FTP_ERROR if there was some other FTP error.
449+ :)
450+declare %an:nondeterministic function
451+ftp:get-binary-nondeterministic( $conn as anyURI, $remote-path as string )
452+ as base64Binary
453+{
454+ ftp:get-binary( $conn, $remote-path )
455+};
456+
457+(:~
458+ : Gets a text file from the FTP server.
459+ :
460+ : @param $conn The opaque URI connection handle previously returned by
461+ : <code>ftp:connect()</code>.
462+ : @param $remote-path The path of the file to get. It must not be empty.
463+ : @param $encoding The character encoding of the file.
464+ : @return the text content of <code>$remote-path</code>.
465+ : @error ftp:INVALID_ARGUMENT if <code>$remote-path</code> is empty
466+ : or <code>$encoding</code> is either an invalid or unsupported encoding.
467+ : @error ftp:NOT_CONNECTED if <code>$conn</code> is either an invalid handle
468+ : or is no longer a valid handle.
469+ : @error ftp:FTP_ERROR if there was some other FTP error.
470+ :)
471+declare function
472+ftp:get-text( $conn as anyURI, $remote-path as string, $encoding as string )
473+ as string external;
474+
475+(:~
476+ : Gets a text file from the FTP server.
477+ :
478+ : @param $conn The opaque URI connection handle previously returned by
479+ : <code>ftp:connect()</code>.
480+ : @param $remote-path The path of the file to get. It must not be empty.
481+ : @param $encoding The character encoding of the file.
482+ : @return the text content of <code>$remote-path</code>.
483+ : @error ftp:INVALID_ARGUMENT if <code>$remote-path</code> is empty
484+ : or <code>$encoding</code> is either an invalid or unsupported encoding.
485+ : @error ftp:NOT_CONNECTED if <code>$conn</code> is either an invalid handle
486+ : or is no longer a valid handle.
487+ : @error ftp:FTP_ERROR if there was some other FTP error.
488+ :)
489+declare %an:nondeterministic function
490+ftp:get-text-nondeterministic( $conn as anyURI, $remote-path as string,
491+ $encoding as string )
492+ as string
493+{
494+ ftp:get-text( $conn, $remote-path, $encoding )
495+};
496+
497+(:~
498+ : Gets a text file, presumed to be encoded in UTF-8, from the FTP server.
499+ :
500+ : @param $conn The opaque URI connection handle previously returned by
501+ : <code>ftp:connect()</code>.
502+ : @param $remote-path The path of the file to get. It must not be empty.
503+ : @return the text content of <code>$remote-path</code>.
504+ : @error ftp:INVALID_ARGUMENT if <code>$remote-path</code> is empty.
505+ : @error ftp:NOT_CONNECTED if <code>$conn</code> is either an invalid handle
506+ : or is no longer a valid handle.
507+ : @error ftp:FTP_ERROR if there was some other FTP error.
508+ :)
509+declare function
510+ftp:get-text( $conn as anyURI, $remote-path as string )
511+ as string
512+{
513+ ftp:get-text( $conn, $remote-path, "UTF-8" )
514+};
515+
516+(:~
517+ : Gets a text file, presumed to be encoded in UTF-8, from the FTP server.
518+ :
519+ : @param $conn The opaque URI connection handle previously returned by
520+ : <code>ftp:connect()</code>.
521+ : @param $remote-path The path of the file to get. It must not be empty.
522+ : @return the text content of <code>$remote-path</code>.
523+ : @error ftp:INVALID_ARGUMENT if <code>$remote-path</code> is empty.
524+ : @error ftp:NOT_CONNECTED if <code>$conn</code> is either an invalid handle
525+ : or is no longer a valid handle.
526+ : @error ftp:FTP_ERROR if there was some other FTP error.
527+ :)
528+declare %an:nondeterministic function
529+ftp:get-text-nondeterministic( $conn as anyURI, $remote-path as string )
530+ as string
531+{
532+ ftp:get-text( $conn, $remote-path, "UTF-8" )
533+};
534+
535+(:~
536+ : Gets a listing for a directory on an FTP server.
537+ :
538+ : @param $conn The opaque URI connection handle previously returned by
539+ : <code>ftp:connect()</code>.
540+ : @param $remote-path The full path of the directory on the FTP server to get
541+ : the directory listing for.
542+ : An empty path is equivalent to <code>/</code>.
543+ : @return Returns a sequence of JSON objects, one per file or subdirectory in
544+ : the listing.
545+ : @error ftp:NOT_CONNECTED if <code>$conn</code> is either an invalid handle
546+ : or is no longer a valid handle.
547+ : @error ftp:FTP_ERROR if there was some other FTP error.
548+ :)
549+declare function
550+ftp:list( $conn as anyURI, $remote-path as string )
551+ as object* external;
552+
553+(:~
554+ : Gets a listing for a directory on an FTP server.
555+ :
556+ : @param $conn The opaque URI connection handle previously returned by
557+ : <code>ftp:connect()</code>.
558+ : @param $remote-path The full path of the directory on the FTP server to get
559+ : the directory listing for.
560+ : An empty path is equivalent to <code>/</code>.
561+ : @return Returns a sequence of JSON objects, one per file or subdirectory in
562+ : the listing.
563+ : @error ftp:NOT_CONNECTED if <code>$conn</code> is either an invalid handle
564+ : or is no longer a valid handle.
565+ : @error ftp:FTP_ERROR if there was some other FTP error.
566+ :)
567+declare %an:nondeterministic function
568+ftp:list-nondeterministic( $conn as anyURI, $remote-path as string )
569+ as object*
570+{
571+ ftp:list( $conn, $remote-path )
572+};
573+
574+(:~
575+ : Creates a directory on the FTP server.
576+ :
577+ : @param $conn The opaque URI connection handle previously returned by
578+ : <code>ftp:connect()</code>.
579+ : @param $remote-path The path of the new directory to create.
580+ : @return the empty sequence.
581+ : @error ftp:INVALID_ARGUMENT if <code>$remote-path</code> is empty.
582+ : @error ftp:NOT_CONNECTED if <code>$conn</code> is either an invalid handle
583+ : or is no longer a valid handle.
584+ : @error ftp:FTP_ERROR if there was some other FTP error.
585+ :)
586+declare %an:sequential function
587+ftp:mkdir( $conn as string, $remote-path as string )
588+ as () external;
589+
590+(:~
591+ : Uploads binary data to a file to the FTP server.
592+ :
593+ : @param $conn The opaque URI connection handle previously returned by
594+ : <code>ftp:connect()</code>.
595+ : @param $binary The binary data to upload.
596+ : @param $remote-path The path of the file to upload to. It must not be empty.
597+ : @return the empty sequence.
598+ : @error ftp:INVALID_ARGUMENT if <code>$remote-path</code> is empty.
599+ : @error ftp:NOT_CONNECTED if <code>$conn</code> is either an invalid handle
600+ : or is no longer a valid handle.
601+ : @error ftp:FTP_ERROR if there was some other FTP error.
602+ :)
603+declare %an:sequential function
604+ftp:put-binary( $conn as anyURI, $binary as base64Binary,
605+ $remote-path as string )
606+ as () external;
607+
608+(:~
609+ : Uploads text to a file to the FTP server.
610+ :
611+ : @param $conn The opaque URI connection handle previously returned by
612+ : <code>ftp:connect()</code>.
613+ : @param $text The text to upload.
614+ : @param $remote-path The path of the file to upload to. It must not be empty.
615+ : @param $encoding The character encoding of the file.
616+ : @return the empty sequence.
617+ : @error ftp:INVALID_ARGUMENT if <code>$remote-path</code> is empty
618+ : or <code>$encoding</code> is either an invalid or unsupported encoding.
619+ : @error ftp:NOT_CONNECTED if <code>$conn</code> is either an invalid handle
620+ : or is no longer a valid handle.
621+ : @error ftp:FTP_ERROR if there was some other FTP error.
622+ :)
623+declare %an:sequential function
624+ftp:put-text( $conn as anyURI, $text as string, $remote-path as string,
625+ $encoding as string )
626+ as () external;
627+
628+(:~
629+ : Uploads text to a UTF-8 encoded file on the FTP server.
630+ :
631+ : @param $conn The opaque URI connection handle previously returned by
632+ : <code>ftp:connect()</code>.
633+ : @param $text The text to upload.
634+ : @param $remote-path The path of the file to upload to. It must not be empty.
635+ : @return the empty sequence.
636+ : @error ftp:INVALID_ARGUMENT if <code>$remote-path</code> is empty.
637+ : @error ftp:NOT_CONNECTED if <code>$conn</code> is either an invalid handle
638+ : or is no longer a valid handle.
639+ : @error ftp:FTP_ERROR if there was some other FTP error.
640+ :)
641+declare %an:sequential function
642+ftp:put-text( $conn as anyURI, $text as string, $remote-path as string )
643+ as ()
644+{
645+ ftp:put-text( $conn, $text, $remote-path, "UTF-8" )
646+};
647+
648+(:~
649+ : Renames a file or directory on an FTP server.
650+ :
651+ : @param $conn The opaque URI connection handle previously returned by
652+ : <code>ftp:connect()</code>.
653+ : @param $remote-from-path The path of the file or directory to rename.
654+ : @param $remote-to-path The new name.
655+ : @return the empty sequence.
656+ : @error ftp:INVALID_ARGUMENT if <code>$remote-from-path</code> or
657+ : <code>$remote-to-path</code> is empty.
658+ : @error ftp:NOT_CONNECTED if <code>$conn</code> is either an invalid handle
659+ : or is no longer a valid handle.
660+ : @error ftp:FTP_ERROR if there was some other FTP error.
661+ :)
662+declare %an:sequential function
663+ftp:rename( $conn as string, $remote-from-path as string,
664+ $remote-to-path as string )
665+ as () external;
666+
667+(:~
668+ : Removes a directory from the FTP server.
669+ :
670+ : @param $conn The opaque URI connection handle previously returned by
671+ : <code>ftp:connect()</code>.
672+ : @param $remote-path The path of the directory to remove.
673+ : @return the empty sequence.
674+ : @error ftp:INVALID_ARGUMENT if <code>$remote-path</code> is empty.
675+ : @error ftp:NOT_CONNECTED if <code>$conn</code> is either an invalid handle
676+ : or is no longer a valid handle.
677+ : @error ftp:FTP_ERROR if there was some other FTP error.
678+ :)
679+declare %an:sequential function
680+ftp:rmdir( $conn as string, $remote-path as string )
681+ as () external;
682+
683+(:===========================================================================:)
684+
685+(: vim:set syntax=xquery et sw=2 ts=2: :)
686
687=== added directory 'modules/ftp-client/ftp-client.xq.src'
688=== added file 'modules/ftp-client/ftp-client.xq.src/ftp_connections.cpp'
689--- modules/ftp-client/ftp-client.xq.src/ftp_connections.cpp 1970-01-01 00:00:00 +0000
690+++ modules/ftp-client/ftp-client.xq.src/ftp_connections.cpp 2014-01-13 19:25:59 +0000
691@@ -0,0 +1,67 @@
692+/*
693+ * Copyright 2006-2013 The FLWOR Foundation.
694+ *
695+ * Licensed under the Apache License, Version 2.0 (the "License");
696+ * you may not use this file except in compliance with the License.
697+ * You may obtain a copy of the License at
698+ *
699+ * http://www.apache.org/licenses/LICENSE-2.0
700+ *
701+ * Unless required by applicable law or agreed to in writing, software
702+ * distributed under the License is distributed on an "AS IS" BASIS,
703+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
704+ * See the License for the specific language governing permissions and
705+ * limitations under the License.
706+ */
707+
708+// standard
709+#include <cassert>
710+
711+// local
712+#include "ftp_connections.h"
713+
714+namespace zorba {
715+namespace ftp_client {
716+
717+///////////////////////////////////////////////////////////////////////////////
718+
719+connections::~connections() {
720+ for ( key_buf_map::const_iterator i = key_buf_.begin();
721+ i != key_buf_.end(); ++i ) {
722+ delete i->second;
723+ }
724+}
725+
726+bool connections::delete_buf( String const &key ) {
727+ key_buf_map::iterator const i( key_buf_.find( key ) );
728+ if ( i != key_buf_.end() ) {
729+ delete i->second;
730+ key_buf_.erase( i );
731+ return true;
732+ }
733+ return false;
734+}
735+
736+void connections::destroy() throw() {
737+ delete this;
738+}
739+
740+curl::streambuf* connections::get_buf( String const &key ) const {
741+ key_buf_map::const_iterator const i( key_buf_.find( key ) );
742+ if ( i != key_buf_.end() )
743+ return i->second;
744+ return 0;
745+}
746+
747+curl::streambuf* connections::new_buf( String const &key ) {
748+ curl::streambuf *&buf = key_buf_[ key ];
749+ assert( !buf );
750+ buf = new curl::streambuf();
751+ return buf;
752+}
753+
754+///////////////////////////////////////////////////////////////////////////////
755+
756+} // namespace ftp_client
757+} // namespace zorba
758+/* vim:set et sw=2 ts=2: */
759
760=== added file 'modules/ftp-client/ftp-client.xq.src/ftp_connections.h'
761--- modules/ftp-client/ftp-client.xq.src/ftp_connections.h 1970-01-01 00:00:00 +0000
762+++ modules/ftp-client/ftp-client.xq.src/ftp_connections.h 2014-01-13 19:25:59 +0000
763@@ -0,0 +1,87 @@
764+/*
765+ * Copyright 2006-2013 The FLWOR Foundation.
766+ *
767+ * Licensed under the Apache License, Version 2.0 (the "License");
768+ * you may not use this file except in compliance with the License.
769+ * You may obtain a copy of the License at
770+ *
771+ * http://www.apache.org/licenses/LICENSE-2.0
772+ *
773+ * Unless required by applicable law or agreed to in writing, software
774+ * distributed under the License is distributed on an "AS IS" BASIS,
775+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
776+ * See the License for the specific language governing permissions and
777+ * limitations under the License.
778+ */
779+
780+#pragma once
781+#ifndef ZORBA_FTP_CLIENT_FTP_CONNECTIONS_H
782+#define ZORBA_FTP_CLIENT_FTP_CONNECTIONS_H
783+
784+// standard
785+#include <map>
786+
787+// Zorba
788+#include <zorba/external_function_parameter.h>
789+#include <zorba/zorba_string.h>
790+
791+// util-curl module
792+#include "util/curl_streambuf.h"
793+
794+namespace zorba {
795+namespace ftp_client {
796+
797+///////////////////////////////////////////////////////////////////////////////
798+
799+/**
800+ * An ftp::connections is-an ExternalFunctionParameter (that's added to a
801+ * query's dynamic context) for keeping a mapping between keys (representing
802+ * URIs of FTP servers) and their associated curl::streambufs (the active
803+ * connections to said servers).
804+ */
805+class connections : public ExternalFunctionParameter {
806+public:
807+ ~connections();
808+
809+ /**
810+ * Deletes a curl::streambuf associatied with the given \a key.
811+ *
812+ * @param key The key associated with the curl::streambuf to delete.
813+ * @return Returns \a true only if \a key was associated with a
814+ * curl::streambuf and therefore deleted; \c false otherwise.
815+ */
816+ bool delete_buf( String const &key );
817+
818+ /**
819+ * Gets an existing curl::streambuf associatied with the given \a key.
820+ *
821+ * @param key The key to get the associated curl::streambuf for.
822+ * @return Returns said curl::streambuf or \c nullptr if \a key has no
823+ * associated curl::streambuf.
824+ */
825+ curl::streambuf* get_buf( String const &key ) const;
826+
827+ /**
828+ * Creates a new curl::streambuf and associates it with the given \a key.
829+ *
830+ * @param key The key to associate the newly created curl::streambuf to.
831+ * The key must not alraedy be associated with any curl::streambuf.
832+ * @return Returns the newly created curl::streambuf.
833+ */
834+ curl::streambuf* new_buf( String const &key );
835+
836+ // inherited
837+ virtual void destroy() throw();
838+
839+private:
840+ // map keys -> cURL::streambuf*
841+ typedef std::map<String,curl::streambuf*> key_buf_map;
842+ key_buf_map key_buf_;
843+};
844+
845+///////////////////////////////////////////////////////////////////////////////
846+
847+} // namespace ftp_client
848+} // namespace zorba
849+#endif /* ZORBA_FTP_CLIENT_FTP_CONNECTIONS_H */
850+/* vim:set et sw=2 ts=2: */
851
852=== added file 'modules/ftp-client/ftp-client.xq.src/ftp_functions.cpp'
853--- modules/ftp-client/ftp-client.xq.src/ftp_functions.cpp 1970-01-01 00:00:00 +0000
854+++ modules/ftp-client/ftp-client.xq.src/ftp_functions.cpp 2014-01-13 19:25:59 +0000
855@@ -0,0 +1,934 @@
856+/*
857+ * Copyright 2006-2013 The FLWOR Foundation.
858+ *
859+ * Licensed under the Apache License, Version 2.0 (the "License");
860+ * you may not use this file except in compliance with the License.
861+ * You may obtain a copy of the License at
862+ *
863+ * http://www.apache.org/licenses/LICENSE-2.0
864+ *
865+ * Unless required by applicable law or agreed to in writing, software
866+ * distributed under the License is distributed on an "AS IS" BASIS,
867+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
868+ * See the License for the specific language governing permissions and
869+ * limitations under the License.
870+ */
871+
872+// standard
873+#include <cctype>
874+#include <ctime>
875+#include <istream>
876+#include <memory>
877+#include <sstream>
878+#include <string>
879+#include <utility> /* for pair */
880+#include <vector>
881+
882+// Zorba
883+#include <zorba/dynamic_context.h>
884+#include <zorba/empty_sequence.h>
885+#include <zorba/item_factory.h>
886+#include <zorba/item_sequence.h>
887+#include <zorba/iterator.h>
888+#include <zorba/singleton_item_sequence.h>
889+#include <zorba/store_consts.h>
890+#include <zorba/user_exception.h>
891+#include <zorba/util/base64_stream.h>
892+#include <zorba/util/mem_streambuf.h>
893+#include <zorba/util/transcode_stream.h>
894+
895+// local
896+#include "ftp_connections.h"
897+#include "ftp_functions.h"
898+#include "ftp_module.h"
899+#include "ftpparse.c"
900+
901+// FTP reply codes; see RFC 959.
902+#define FTP_REPLY_ACTION_COMPLETED 250
903+#define FTP_REPLY_PATH_CREATED 257
904+#define FTP_REPLY_ACTION_ABORTED 451
905+#define FTP_REPLY_ACTION_NOT_TAKEN 550
906+
907+#define IS_ATOMIC_TYPE(ITEM,TYPE) \
908+ ( (ITEM).isAtomic() && (ITEM).getTypeCode() == store::TYPE )
909+
910+// Eliminates "control may reach end of non-void function" warning.
911+#define THROW_EXCEPTION(...) while (1) throw_exception( __VA_ARGS__ )
912+
913+/**
914+ * This is our "external function parameter" name in the dynamic context.
915+ */
916+#define ZORBA_FTP_CONNECTIONS "http://zorba.io/modules/ftp-client/connections"
917+
918+using namespace std;
919+
920+namespace zorba {
921+namespace ftp_client {
922+
923+///////////////////////////////////////////////////////////////////////////////
924+
925+struct curl_helper {
926+ curl_helper( curl::streambuf*, curl_slist* = 0 );
927+ ~curl_helper();
928+private:
929+ curl::streambuf *const cbuf_;
930+ curl_slist *const slist_;
931+};
932+
933+curl_helper::curl_helper( curl::streambuf *cbuf, curl_slist *slist ) :
934+ cbuf_( cbuf ),
935+ slist_( slist )
936+{
937+ ZORBA_CURLM_ASSERT( curl_multi_remove_handle( cbuf_->curlm(), cbuf_->curl() ) );
938+}
939+
940+curl_helper::~curl_helper() {
941+ if ( slist_ )
942+ curl_slist_free_all( slist_ );
943+ CURL *const cobj = cbuf_->curl();
944+ curl_easy_setopt( cobj, CURLOPT_CUSTOMREQUEST, NULL );
945+ curl_easy_setopt( cobj, CURLOPT_HEADERDATA, NULL );
946+ curl_easy_setopt( cobj, CURLOPT_HEADERFUNCTION, NULL );
947+ curl_easy_setopt( cobj, CURLOPT_UPLOAD, 0L );
948+ curl_multi_add_handle( cbuf_->curlm(), cobj );
949+}
950+
951+///////////////////////////////////////////////////////////////////////////////
952+
953+static connections& get_connections( DynamicContext const *dctx ) {
954+ connections *conns = static_cast<connections*>(
955+ dctx->getExternalFunctionParameter( ZORBA_FTP_CONNECTIONS )
956+ );
957+ if ( !conns ) {
958+ conns = new connections();
959+ dctx->addExternalFunctionParameter( ZORBA_FTP_CONNECTIONS, conns );
960+ }
961+ return *conns;
962+}
963+
964+static int get_ftp_reply_code( CURL *cobj ) {
965+ long code;
966+ ZORBA_CURL_ASSERT( curl_easy_getinfo( cobj, CURLINFO_RESPONSE_CODE, &code ) );
967+ return static_cast<int>( code );
968+}
969+
970+inline int get_scheme_len( String const &uri ) {
971+ if ( uri.compare( 0, 6, "ftp://" ) == 0 )
972+ return 6;
973+ if ( uri.compare( 0, 7, "ftps://" ) == 0 )
974+ return 7;
975+ return 0;
976+}
977+
978+static String host_part( String /* intentionally not const& */ uri ) {
979+ if ( int const scheme_len = get_scheme_len( uri ) ) {
980+ uri.erase( 0, scheme_len );
981+ String::size_type const at_pos = uri.find( '@' );
982+ if ( at_pos != String::npos )
983+ uri.erase( 0, at_pos + 1 ); // erase user:password@
984+ String::size_type const colon_pos = uri.find( ':' );
985+ if ( colon_pos != String::npos )
986+ uri.erase( colon_pos ); // erase :port
987+ }
988+ return uri;
989+}
990+
991+static String make_uri( String const &uri,
992+ String path, // intentionally not const&
993+ bool path_is_dir = false ) {
994+ if ( path.empty() )
995+ path = '/';
996+ else {
997+ if ( path_is_dir && path[ path.size() - 1 ] != '/' )
998+ path += '/';
999+ if ( path[0] != '/' )
1000+ path.insert( (String::size_type)0, 1, '/' );
1001+ }
1002+ String result( uri );
1003+ result += path;
1004+ return result;
1005+}
1006+
1007+static void stream_releaser( istream *is ) {
1008+ delete is;
1009+}
1010+
1011+///////////////////////////////////////////////////////////////////////////////
1012+
1013+function::function( module const *m, char const *local_name ) :
1014+ module_( m ),
1015+ local_name_( local_name )
1016+{
1017+}
1018+
1019+bool function::get_bool_opt( Item const &options, char const *key,
1020+ bool default_value ) const {
1021+ Item const item( options.getObjectValue( key ) );
1022+ if ( item.isNull() )
1023+ return default_value;
1024+ if ( !IS_ATOMIC_TYPE( item, XS_BOOLEAN ) )
1025+ THROW_EXCEPTION( "INVALID_ARGUMENT", key, "value must be boolean" );
1026+ return item.getBooleanValue();
1027+}
1028+
1029+int function::get_integer_opt( Item const &options, char const *key,
1030+ int default_value ) const {
1031+ Item const item( options.getObjectValue( key ) );
1032+ if ( item.isNull() )
1033+ return default_value;
1034+ if ( !IS_ATOMIC_TYPE( item, XS_INTEGER ) )
1035+ THROW_EXCEPTION( "INVALID_ARGUMENT", key, "value must be integer" );
1036+ return item.getIntValue();
1037+}
1038+
1039+Item function::get_item_arg( ExternalFunction::Arguments_t const &args,
1040+ unsigned pos ) const {
1041+ Item result;
1042+ if ( pos < args.size() ) {
1043+ Iterator_t it( args[ pos ]->getIterator() );
1044+ it->open();
1045+ it->next( result );
1046+ it->close();
1047+ }
1048+ return result;
1049+}
1050+
1051+String function::getLocalName() const {
1052+ return local_name_;
1053+}
1054+
1055+String function::get_string_arg( ExternalFunction::Arguments_t const &args,
1056+ unsigned pos ) const {
1057+ String s;
1058+ Item const item( get_item_arg( args, pos ) );
1059+ if ( !item.isNull() )
1060+ s = item.getStringValue();
1061+ return s;
1062+}
1063+
1064+String function::get_string_opt( Item const &options, char const *key,
1065+ char const *default_value ) const {
1066+ Item const item( options.getObjectValue( key ) );
1067+ if ( item.isNull() )
1068+ return default_value;
1069+ if ( !IS_ATOMIC_TYPE( item, XS_STRING ) )
1070+ THROW_EXCEPTION( "INVALID_ARGUMENT", key, "value must be string" );
1071+ return item.getStringValue();
1072+}
1073+
1074+String function::getURI() const {
1075+ return module_->getURI();
1076+}
1077+
1078+curl::streambuf* function::require_connection( DynamicContext const *dctx,
1079+ String const &conn ) const {
1080+ connections &conns = get_connections( dctx );
1081+ if ( curl::streambuf *const cbuf = conns.get_buf( conn.c_str() ) )
1082+ return cbuf;
1083+ THROW_EXCEPTION( "NOT_CONNECTED", host_part( conn ), "not connnected" );
1084+}
1085+
1086+void function::throw_exception( char const *error_code, char const *object,
1087+ char const *message, int ftp_code ) const {
1088+ string s;
1089+
1090+ if ( object && *object ) {
1091+ ostringstream oss;
1092+ oss << '"' << object << "\": " << message;
1093+ s = oss.str();
1094+ } else
1095+ s = message;
1096+
1097+ if ( ftp_code ) {
1098+ ostringstream oss;
1099+ oss << " (FTP code " << ftp_code << ')';
1100+ s += oss.str();
1101+ }
1102+
1103+ throw USER_EXCEPTION(
1104+ module_->getItemFactory()->createQName( getURI(), error_code ), s
1105+ );
1106+}
1107+
1108+///////////////////////////////////////////////////////////////////////////////
1109+
1110+get_function::get_function( module const *m, char const *local_name,
1111+ bool text ) :
1112+ function( m, local_name ),
1113+ text_( text )
1114+{
1115+}
1116+
1117+ItemSequence_t
1118+get_function::evaluate( ExternalFunction::Arguments_t const &args,
1119+ StaticContext const*,
1120+ DynamicContext const *dctx ) const {
1121+ String const conn( get_string_arg( args, 0 ) );
1122+ String const path( get_string_arg( args, 1 ) );
1123+ if ( path.empty() )
1124+ THROW_EXCEPTION( "INVALID_ARGUMENT", "", "empty path" );
1125+ String const encoding( text_ ? get_string_arg( args, 2 ) : "" );
1126+ if ( !encoding.empty() && transcode::is_necessary( encoding.c_str() ) &&
1127+ !transcode::is_supported( encoding.c_str() ) ) {
1128+ THROW_EXCEPTION( "INVALID_ARGUMENT", encoding, "unsupported encoding" );
1129+ }
1130+ String const uri( make_uri( conn, path ) );
1131+
1132+ curl::streambuf *const cbuf = require_connection( dctx, conn );
1133+ CURL *const cobj = cbuf->curl();
1134+ curl_easy_setopt( cobj, CURLOPT_TRANSFERTEXT, text_ ? 1 : 0 );
1135+ curl_easy_setopt( cobj, CURLOPT_URL, uri.c_str() );
1136+
1137+ istream *const is = new istream( cbuf );
1138+ if ( transcode::is_necessary( encoding.c_str() ) )
1139+ transcode::attach( *is, encoding.c_str() );
1140+
1141+ ItemFactory *const f = module_->getItemFactory();
1142+ Item result(
1143+ text_ ?
1144+ f->createStreamableString( *is, &stream_releaser )
1145+ :
1146+ f->createStreamableBase64Binary( *is, &stream_releaser, false )
1147+ );
1148+ return ItemSequence_t( new SingletonItemSequence( result ) );
1149+}
1150+
1151+///////////////////////////////////////////////////////////////////////////////
1152+
1153+static size_t curl_read_callback( void *ptr, size_t size, size_t nmemb,
1154+ void *data ) {
1155+ size *= nmemb;
1156+ istream *const is = static_cast<istream*>( data );
1157+ is->read( static_cast<char*>( ptr ), static_cast<streamsize>( size ) );
1158+ return is->gcount();
1159+}
1160+
1161+put_function::put_function( module const *m, char const *local_name,
1162+ bool text ) :
1163+ function( m, local_name ),
1164+ text_( text )
1165+{
1166+}
1167+
1168+ItemSequence_t
1169+put_function::evaluate( ExternalFunction::Arguments_t const &args,
1170+ StaticContext const*,
1171+ DynamicContext const *dctx ) const {
1172+ String const conn( get_string_arg( args, 0 ) );
1173+ Item put_item( get_item_arg( args, 1 ) );
1174+ String const path( get_string_arg( args, 2 ) );
1175+ if ( path.empty() )
1176+ THROW_EXCEPTION( "INVALID_ARGUMENT", "", "empty path" );
1177+ String const encoding( text_ ? get_string_arg( args, 2 ) : "" );
1178+ if ( !encoding.empty() && transcode::is_necessary( encoding.c_str() ) &&
1179+ !transcode::is_supported( encoding.c_str() ) ) {
1180+ THROW_EXCEPTION( "INVALID_ARGUMENT", encoding, "unsupported encoding" );
1181+ }
1182+ String const uri( make_uri( conn, path ) );
1183+
1184+ curl::streambuf *const cbuf = require_connection( dctx, conn );
1185+ CURL *const cobj = cbuf->curl();
1186+ ZORBA_CURL_ASSERT( curl_easy_setopt( cobj, CURLOPT_TRANSFERTEXT, text_ ? 1 : 0 ) );
1187+ ZORBA_CURL_ASSERT( curl_easy_setopt( cobj, CURLOPT_URL, uri.c_str() ) );
1188+
1189+ istream *is;
1190+ mem_streambuf mbuf;
1191+ auto_ptr<istream> raii_is;
1192+ String text;
1193+
1194+ if ( put_item.isStreamable() ) {
1195+ is = &put_item.getStream();
1196+ } else {
1197+ if ( text_ ) {
1198+ text = put_item.getStringValue();
1199+ mbuf.set( const_cast<char*>( text.data() ), text.size() );
1200+ } else {
1201+ size_t size;
1202+ char const *const data = put_item.getBase64BinaryValue( size );
1203+ mbuf.set( const_cast<char*>( data ), size );
1204+ }
1205+ is = new istream( &mbuf );
1206+ raii_is.reset( is );
1207+ }
1208+
1209+ //
1210+ // Thest raii_* objects must be constructed after raii_is so their destructor
1211+ // is called before that of raii_is (reverse order of construction) so either
1212+ // the base64_streambuf or transcode_streambuf is destroyed before the stream
1213+ // it's attached to is.
1214+ //
1215+ base64::auto_attach<istream> raii_b64;
1216+ transcode::auto_attach<istream> raii_xcode;
1217+ if ( text_ ) {
1218+ if ( !encoding.empty() && transcode::is_necessary( encoding.c_str() ) )
1219+ raii_xcode.attach( *is, encoding.c_str() );
1220+ } else {
1221+ if ( put_item.isEncoded() )
1222+ raii_b64.attach( *is );
1223+ }
1224+
1225+ try {
1226+ ZORBA_CURL_ASSERT( curl_easy_setopt( cobj, CURLOPT_READDATA, is ) );
1227+ ZORBA_CURL_ASSERT( curl_easy_setopt( cobj, CURLOPT_READFUNCTION, curl_read_callback ) );
1228+ ZORBA_CURL_ASSERT( curl_easy_setopt( cobj, CURLOPT_UPLOAD, 1L ) );
1229+ curl_helper helper( cbuf );
1230+ ZORBA_CURL_ASSERT( curl_easy_perform( cobj ) );
1231+ return ItemSequence_t( new EmptySequence() );
1232+ }
1233+ catch ( std::exception const &e ) {
1234+ THROW_EXCEPTION( "FTP_ERROR", path, e.what() );
1235+ }
1236+}
1237+
1238+///////////////////////////////////////////////////////////////////////////////
1239+
1240+connect_function::connect_function( module const *m ) :
1241+ function( m, "connect" )
1242+{
1243+}
1244+
1245+ItemSequence_t
1246+connect_function::evaluate( ExternalFunction::Arguments_t const &args,
1247+ StaticContext const*,
1248+ DynamicContext const *dctx ) const {
1249+ String const uri( get_string_arg( args, 0 ) );
1250+ String conn( uri );
1251+ //
1252+ // Even though we allow ftp URIs ("ftp://host"), we limit them to refer only
1253+ // to the host and not either a file or subdirectory; hence chop off
1254+ // everything starting at the first '/'.
1255+ //
1256+ int const scheme_len = get_scheme_len( conn );
1257+ String::size_type const slash_pos = conn.find( scheme_len, '/' );
1258+ if ( slash_pos != String::npos )
1259+ conn.erase( slash_pos );
1260+
1261+ Item const options( get_item_arg( args, 1 ) );
1262+ String const password( get_string_opt( options, "password" ) );
1263+ int const port( get_integer_opt( options, "port" ) );
1264+ String const protocol( get_string_opt( options, "protocol", "ftp" ) );
1265+ String const ssl_comm( get_string_opt( options, "SSL-communication" ) );
1266+ bool const ssl_verify( get_bool_opt( options, "SSL-verify", true ) );
1267+ bool const trace( get_bool_opt( options, "trace", false ) );
1268+ String const user( get_string_opt( options, "user" ) );
1269+
1270+ if ( !(protocol == "ftp" || protocol == "ftps") )
1271+ THROW_EXCEPTION(
1272+ "INVALID_ARGUMENT", "protcol", "must be either ftp or ftps"
1273+ );
1274+
1275+ long use_ssl;
1276+ if ( !ssl_comm.empty() ) {
1277+ if ( ssl_comm == "all" )
1278+ use_ssl = CURLUSESSL_ALL;
1279+ else if ( ssl_comm == "control" )
1280+ use_ssl = CURLUSESSL_CONTROL;
1281+ else if ( ssl_comm == "none" )
1282+ use_ssl = CURLUSESSL_NONE;
1283+ else if ( ssl_comm == "try" )
1284+ use_ssl = CURLUSESSL_TRY;
1285+ else
1286+ THROW_EXCEPTION(
1287+ "INVALID_ARGUMENT", "SSL-communication",
1288+ "must be one of: none, try, control, or all"
1289+ );
1290+ } else if ( protocol == "ftps" )
1291+ use_ssl = CURLUSESSL_ALL;
1292+ else
1293+ use_ssl = CURLUSESSL_NONE;
1294+
1295+ if ( !scheme_len ) {
1296+ if ( user.empty() && !password.empty() )
1297+ THROW_EXCEPTION(
1298+ "INVALID_ARGUMENT", "", "empty user and non-empty password"
1299+ );
1300+ if ( !user.empty() ) {
1301+ conn.insert( (String::size_type)0, 1, '@' );
1302+ if ( !password.empty() ) {
1303+ char *const esc_password =
1304+ curl_escape( const_cast<char*>( password.data() ), password.size() );
1305+ conn.insert( 0, esc_password );
1306+ curl_free( esc_password );
1307+ conn.insert( (String::size_type)0, 1, ':' );
1308+ }
1309+ conn.insert( 0, user );
1310+ }
1311+ conn.insert( 0, "://" );
1312+ conn.insert( 0, protocol );
1313+ if ( port ) {
1314+ conn.append( 1, ':' );
1315+ ostringstream oss;
1316+ oss << port;
1317+ conn.append( oss.str() );
1318+ }
1319+ }
1320+
1321+ connections &conns = get_connections( dctx );
1322+ curl::streambuf *cbuf = conns.get_buf( conn );
1323+ if ( cbuf )
1324+ THROW_EXCEPTION(
1325+ "ALREADY_CONNECTED", uri, "connection previously established"
1326+ );
1327+ cbuf = conns.new_buf( conn );
1328+
1329+ try {
1330+ cbuf->open( conn.c_str() );
1331+ CURL *const cobj = cbuf->curl();
1332+
1333+ if ( trace )
1334+ cbuf->curl_verbose( true );
1335+ ZORBA_CURL_ASSERT( curl_easy_setopt( cobj, CURLOPT_USE_SSL, use_ssl ) );
1336+ if ( !ssl_verify ) {
1337+ ZORBA_CURL_ASSERT( curl_easy_setopt( cobj, CURLOPT_SSL_VERIFYHOST, 0L ) );
1338+ ZORBA_CURL_ASSERT( curl_easy_setopt( cobj, CURLOPT_SSL_VERIFYPEER, 0L ) );
1339+ }
1340+
1341+ curl_helper helper( cbuf );
1342+ ZORBA_CURL_ASSERT( curl_easy_perform( cobj ) );
1343+ Item result( module_->getItemFactory()->createAnyURI( conn ) );
1344+ return ItemSequence_t( new SingletonItemSequence( result ) );
1345+ }
1346+ catch ( std::exception const &e ) {
1347+ THROW_EXCEPTION( "FTP_ERROR", uri, e.what() );
1348+ }
1349+}
1350+
1351+///////////////////////////////////////////////////////////////////////////////
1352+
1353+#if ZORBA_FTP_CWD_IMPLEMENTED
1354+ItemSequence_t
1355+cwd_function::evaluate( ExternalFunction::Arguments_t const &args,
1356+ StaticContext const*,
1357+ DynamicContext const *dctx ) const {
1358+ String const conn( get_string_arg( args, 0 ) );
1359+ String const path( get_string_arg( args, 1 ) );
1360+ if ( path.empty() )
1361+ THROW_EXCEPTION( "INVALID_ARGUMENT", "", "empty path" );
1362+
1363+ curl::streambuf *const cbuf = require_connection( dctx, conn );
1364+ CURL *const cobj = cbuf->curl();
1365+
1366+ String req( "CWD " );
1367+ req += path;
1368+ curl_easy_setopt( cobj, CURLOPT_CUSTOMREQUEST, req.c_str() );
1369+ curl_easy_perform( cobj );
1370+
1371+ return ItemSequence_t( new EmptySequence() );
1372+}
1373+#endif
1374+
1375+///////////////////////////////////////////////////////////////////////////////
1376+
1377+delete_function::delete_function( module const *m ) :
1378+ function( m, "delete" )
1379+{
1380+}
1381+
1382+ItemSequence_t
1383+delete_function::evaluate( ExternalFunction::Arguments_t const &args,
1384+ StaticContext const*,
1385+ DynamicContext const *dctx ) const {
1386+ String const conn( get_string_arg( args, 0 ) );
1387+ String const path( get_string_arg( args, 1 ) );
1388+ if ( path.empty() )
1389+ THROW_EXCEPTION( "INVALID_ARGUMENT", "", "empty path" );
1390+ String const command( "DELE " + path );
1391+
1392+ curl::streambuf *const cbuf = require_connection( dctx, conn );
1393+ CURL *const cobj = cbuf->curl();
1394+ curl_easy_setopt( cobj, CURLOPT_CUSTOMREQUEST, command.c_str() );
1395+
1396+ try {
1397+ curl_helper helper( cbuf );
1398+ ZORBA_CURL_ASSERT( curl_easy_perform( cobj ) );
1399+ }
1400+ catch ( curl::exception const &e ) {
1401+ int const ftp_code = get_ftp_reply_code( cobj );
1402+ switch ( ftp_code ) {
1403+ case FTP_REPLY_ACTION_COMPLETED:
1404+ if ( e.curl_code() == CURLE_FTP_COULDNT_RETR_FILE ) {
1405+ //
1406+ // After the file is deleted, CURL tries to RETR it which fails, so
1407+ // ignore that error.
1408+ // (Also see <http://stackoverflow.com/a/13515807/99089>.)
1409+ //
1410+ break;
1411+ }
1412+ THROW_EXCEPTION( "FTP_ERROR", path, e.what() );
1413+ case FTP_REPLY_ACTION_NOT_TAKEN:
1414+ THROW_EXCEPTION( "FTP_ERROR", path, "file not found", ftp_code );
1415+ default:
1416+ THROW_EXCEPTION( "FTP_ERROR", path, e.what(), ftp_code );
1417+ } // switch
1418+ }
1419+ catch ( std::exception const &e ) {
1420+ THROW_EXCEPTION( "FTP_ERROR", path, e.what() );
1421+ }
1422+ return ItemSequence_t( new EmptySequence() );
1423+}
1424+
1425+///////////////////////////////////////////////////////////////////////////////
1426+
1427+#if 0
1428+disconnect_function::disconnect_function( module const *m ) :
1429+ function( m, "disconnect" )
1430+{
1431+}
1432+
1433+ItemSequence_t
1434+disconnect_function::evaluate( ExternalFunction::Arguments_t const &args,
1435+ StaticContext const*,
1436+ DynamicContext const *dctx ) const {
1437+ String const conn( get_string_arg( args, 0 ) );
1438+ connections &conns = get_connections( dctx );
1439+ if ( !conns.delete_buf( conn ) )
1440+ THROW_EXCEPTION( "NOT_CONNECTED", host_part( conn ), "not connected" );
1441+ return ItemSequence_t( new EmptySequence() );
1442+}
1443+#endif
1444+
1445+///////////////////////////////////////////////////////////////////////////////
1446+
1447+get_binary_function::get_binary_function( module const *m ) :
1448+ get_function( m, "get-binary", false )
1449+{
1450+}
1451+
1452+///////////////////////////////////////////////////////////////////////////////
1453+
1454+get_text_function::get_text_function( module const *m ) :
1455+ get_function( m, "get-text", true )
1456+{
1457+}
1458+
1459+///////////////////////////////////////////////////////////////////////////////
1460+
1461+class list_iterator : public ItemSequence, public Iterator {
1462+public:
1463+ list_iterator( curl::streambuf*, ItemFactory* );
1464+
1465+ // inherited from ItemSequence
1466+ Iterator_t getIterator();
1467+
1468+ // inherited from Iterator
1469+ void close();
1470+ bool isOpen() const;
1471+ bool next( Item& );
1472+ void open();
1473+
1474+private:
1475+ ItemFactory *const factory_;
1476+ istream is_;
1477+ bool open_;
1478+};
1479+
1480+list_iterator::list_iterator( curl::streambuf *cbuf, ItemFactory *factory ) :
1481+ factory_( factory ),
1482+ is_( cbuf ),
1483+ open_( false )
1484+{
1485+}
1486+
1487+void list_iterator::close() {
1488+ open_ = false;
1489+}
1490+
1491+Iterator_t list_iterator::getIterator() {
1492+ return this;
1493+}
1494+
1495+bool list_iterator::isOpen() const {
1496+ return open_;
1497+}
1498+
1499+bool list_iterator::next( Item &result ) {
1500+ static Item const mtime_key( factory_->createString( "mtime" ) );
1501+ static Item const name_key( factory_->createString( "name" ) );
1502+ static Item const size_key( factory_->createString( "size" ) );
1503+
1504+ string line;
1505+ while ( getline( is_, line ) ) {
1506+ if ( line.empty() )
1507+ continue;
1508+ if ( line[ line.size() - 1 ] == '\r' )
1509+ line.erase( line.size() - 1 );
1510+ struct ftpparse ftp_file;
1511+ if ( ftpparse( &ftp_file, line.data(), line.size() ) ) {
1512+ vector<pair<Item,Item> > kv;
1513+
1514+ String name( ftp_file.name, ftp_file.namelen );
1515+ Item const name_value( factory_->createString( name ) );
1516+ kv.push_back( make_pair( name_key, name_value ) );
1517+
1518+ switch ( ftp_file.sizetype ) {
1519+ case FTPPARSE_SIZE_ASCII:
1520+ case FTPPARSE_SIZE_BINARY: {
1521+ Item const size_value( factory_->createLong( ftp_file.size ) );
1522+ kv.push_back( make_pair( size_key, size_value ) );
1523+ break;
1524+ }
1525+ case FTPPARSE_SIZE_UNKNOWN:
1526+ // do nothing
1527+ break;
1528+ } // switch
1529+
1530+ struct tm tm;
1531+ gmtime_r( &ftp_file.mtime, &tm );
1532+ int const year = tm.tm_year + 1900;
1533+ switch ( ftp_file.mtimetype ) {
1534+ case FTPPARSE_MTIME_REMOTEDAY:
1535+ tm.tm_hour = tm.tm_min = 0;
1536+ // no break;
1537+ case FTPPARSE_MTIME_REMOTEMINUTE:
1538+ tm.tm_sec = 0;
1539+ tm.tm_gmtoff = 0;
1540+ // no break;
1541+ case FTPPARSE_MTIME_LOCAL: {
1542+ Item const mtime_value (
1543+ factory_->createDateTime(
1544+ year, tm.tm_mon, tm.tm_mday,
1545+ tm.tm_hour, tm.tm_min, tm.tm_sec, (int)tm.tm_gmtoff
1546+ )
1547+ );
1548+ kv.push_back( make_pair( mtime_key, mtime_value ) );
1549+ // no break;
1550+ }
1551+ case FTPPARSE_MTIME_UNKNOWN:
1552+ // do nothing
1553+ break;
1554+ } // switch
1555+
1556+ result = factory_->createJSONObject( kv );
1557+ return true;
1558+ } // if
1559+ } // while
1560+ return false;
1561+}
1562+
1563+void list_iterator::open() {
1564+ open_ = true;
1565+}
1566+
1567+list_function::list_function( module const *m ) :
1568+ function( m, "list" )
1569+{
1570+}
1571+
1572+ItemSequence_t
1573+list_function::evaluate( ExternalFunction::Arguments_t const &args,
1574+ StaticContext const*,
1575+ DynamicContext const *dctx ) const {
1576+ String const conn( get_string_arg( args, 0 ) );
1577+ String const path( get_string_arg( args, 1 ) );
1578+ String const uri( make_uri( conn, path, true ) );
1579+
1580+ curl::streambuf *const cbuf = require_connection( dctx, conn );
1581+ CURL *const cobj = cbuf->curl();
1582+ ZORBA_CURL_ASSERT( curl_easy_setopt( cobj, CURLOPT_URL, uri.c_str() ) );
1583+ try {
1584+ return ItemSequence_t(
1585+ new list_iterator( cbuf, module_->getItemFactory() )
1586+ );
1587+ }
1588+ catch ( std::exception const &e ) {
1589+ THROW_EXCEPTION( "FTP_ERROR", path, e.what() );
1590+ }
1591+}
1592+
1593+///////////////////////////////////////////////////////////////////////////////
1594+
1595+mkdir_function::mkdir_function( module const *m ) :
1596+ function( m, "mkdir" )
1597+{
1598+}
1599+
1600+ItemSequence_t
1601+mkdir_function::evaluate( ExternalFunction::Arguments_t const &args,
1602+ StaticContext const*,
1603+ DynamicContext const *dctx ) const {
1604+ String const conn( get_string_arg( args, 0 ) );
1605+ String const path( get_string_arg( args, 1 ) );
1606+ if ( path.empty() )
1607+ THROW_EXCEPTION( "INVALID_ARGUMENT", "", "empty path" );
1608+ String const command( "MKD " + path );
1609+
1610+ curl::streambuf *const cbuf = require_connection( dctx, conn );
1611+ CURL *const cobj = cbuf->curl();
1612+ curl_easy_setopt( cobj, CURLOPT_CUSTOMREQUEST, command.c_str() );
1613+
1614+ try {
1615+ curl_helper helper( cbuf );
1616+ ZORBA_CURL_ASSERT( curl_easy_perform( cobj ) );
1617+ }
1618+ catch ( curl::exception const &e ) {
1619+ int const ftp_code = get_ftp_reply_code( cobj );
1620+ switch ( ftp_code ) {
1621+ case FTP_REPLY_ACTION_NOT_TAKEN:
1622+ THROW_EXCEPTION( "FTP_ERROR", path, "directory exists", ftp_code );
1623+ case FTP_REPLY_PATH_CREATED:
1624+ if ( e.curl_code() == CURLE_FTP_COULDNT_RETR_FILE ) {
1625+ //
1626+ // After the directory is created, CURL tries to RETR it which fails,
1627+ // so ignore that error.
1628+ // (Also see <http://stackoverflow.com/a/13515807/99089>.)
1629+ //
1630+ break;
1631+ }
1632+ THROW_EXCEPTION( "FTP_ERROR", path, e.what() );
1633+ default:
1634+ THROW_EXCEPTION( "FTP_ERROR", path, e.what(), ftp_code );
1635+ } // switch
1636+ }
1637+ catch ( std::exception const &e ) {
1638+ THROW_EXCEPTION( "FTP_ERROR", path, e.what() );
1639+ }
1640+ return ItemSequence_t( new EmptySequence() );
1641+}
1642+
1643+///////////////////////////////////////////////////////////////////////////////
1644+
1645+put_binary_function::put_binary_function( module const *m ) :
1646+ put_function( m, "put-binary", false )
1647+{
1648+}
1649+
1650+///////////////////////////////////////////////////////////////////////////////
1651+
1652+put_text_function::put_text_function( module const *m ) :
1653+ put_function( m, "put-text", true )
1654+{
1655+}
1656+
1657+///////////////////////////////////////////////////////////////////////////////
1658+
1659+static size_t curl_header_callback( void *ptr, size_t size, size_t nmemb,
1660+ void *data ) {
1661+ size *= nmemb;
1662+ String *const ftp_reply_msg = static_cast<String*>( data );
1663+
1664+ // skip 3-digit reply code and 1 space
1665+ char const *s = static_cast<char*>( ptr ) + 4;
1666+ size_t len = size - 4;
1667+
1668+ // trim trailing whitespace
1669+ while ( len && isspace( s[ len - 1 ] ) )
1670+ --len;
1671+
1672+ ftp_reply_msg->assign( s, len );
1673+ return size; // must always return original size
1674+}
1675+
1676+rename_function::rename_function( module const *m ) :
1677+ function( m, "rename" )
1678+{
1679+}
1680+
1681+ItemSequence_t
1682+rename_function::evaluate( ExternalFunction::Arguments_t const &args,
1683+ StaticContext const*,
1684+ DynamicContext const *dctx ) const {
1685+ String const conn( get_string_arg( args, 0 ) );
1686+ String const from_path( get_string_arg( args, 1 ) );
1687+ if ( from_path.empty() )
1688+ THROW_EXCEPTION( "INVALID_ARGUMENT", "", "\"from\" path empty" );
1689+ String const to_path( get_string_arg( args, 2 ) );
1690+ if ( to_path.empty() )
1691+ THROW_EXCEPTION( "INVALID_ARGUMENT", "", "\to\" path empty" );
1692+ String const command1( "RNFR " + from_path );
1693+ String const command2( "RNTO " + to_path );
1694+
1695+ curl::streambuf *const cbuf = require_connection( dctx, conn );
1696+ CURL *const cobj = cbuf->curl();
1697+
1698+ curl_slist *slist = 0;
1699+ slist = curl_slist_append( slist, command1.c_str() );
1700+ slist = curl_slist_append( slist, command2.c_str() );
1701+ curl_easy_setopt( cobj, CURLOPT_QUOTE, slist );
1702+
1703+ //
1704+ // When doing a rename, there isn't a 1-to-1 mapping between an FTP reply
1705+ // code and a specific error; hence we can't use our own error message.
1706+ // Instead, we capture the FTP replies and parse the messages out of them.
1707+ // We actually only care about the last one since that's the one for the
1708+ // error. We then use that for the error message.
1709+ //
1710+ curl_easy_setopt( cobj, CURLOPT_HEADERFUNCTION, curl_header_callback );
1711+ String ftp_reply_msg;
1712+ curl_easy_setopt( cobj, CURLOPT_HEADERDATA, &ftp_reply_msg );
1713+
1714+ try {
1715+ curl_helper helper( cbuf, slist );
1716+ ZORBA_CURL_ASSERT( curl_easy_perform( cobj ) );
1717+ }
1718+ catch ( curl::exception const &e ) {
1719+ int const ftp_code = get_ftp_reply_code( cobj );
1720+ switch ( ftp_code ) {
1721+ case FTP_REPLY_ACTION_ABORTED:
1722+ THROW_EXCEPTION( "FTP_ERROR", to_path, ftp_reply_msg, ftp_code );
1723+ case FTP_REPLY_ACTION_NOT_TAKEN:
1724+ THROW_EXCEPTION( "FTP_ERROR", from_path, ftp_reply_msg, ftp_code );
1725+ default:
1726+ THROW_EXCEPTION( "FTP_ERROR", from_path, e.what(), ftp_code );
1727+ } // switch
1728+ }
1729+ catch ( std::exception const &e ) {
1730+ THROW_EXCEPTION( "FTP_ERROR", from_path, e.what() );
1731+ }
1732+ return ItemSequence_t( new EmptySequence() );
1733+}
1734+
1735+///////////////////////////////////////////////////////////////////////////////
1736+
1737+rmdir_function::rmdir_function( module const *m ) :
1738+ function( m, "rmdir" )
1739+{
1740+}
1741+
1742+ItemSequence_t
1743+rmdir_function::evaluate( ExternalFunction::Arguments_t const &args,
1744+ StaticContext const*,
1745+ DynamicContext const *dctx ) const {
1746+ String const conn( get_string_arg( args, 0 ) );
1747+ String const path( get_string_arg( args, 1 ) );
1748+ if ( path.empty() )
1749+ THROW_EXCEPTION( "INVALID_ARGUMENT", "", "empty path" );
1750+ String const command( "RMD " + path );
1751+
1752+ curl::streambuf *const cbuf = require_connection( dctx, conn );
1753+ CURL *const cobj = cbuf->curl();
1754+ curl_easy_setopt( cobj, CURLOPT_CUSTOMREQUEST, command.c_str() );
1755+
1756+ try {
1757+ curl_helper helper( cbuf );
1758+ ZORBA_CURL_ASSERT( curl_easy_perform( cobj ) );
1759+ }
1760+ catch ( curl::exception const &e ) {
1761+ int const ftp_code = get_ftp_reply_code( cobj );
1762+ switch ( ftp_code ) {
1763+ case FTP_REPLY_ACTION_COMPLETED:
1764+ if ( e.curl_code() == CURLE_FTP_COULDNT_RETR_FILE ) {
1765+ //
1766+ // After the directory is deleted, CURL tries to RETR it which fails,
1767+ // so ignore that error.
1768+ // (Also see <http://stackoverflow.com/a/13515807/99089>.)
1769+ //
1770+ break;
1771+ }
1772+ THROW_EXCEPTION( "FTP_ERROR", path, e.what() );
1773+ case FTP_REPLY_ACTION_NOT_TAKEN:
1774+ THROW_EXCEPTION( "FTP_ERROR", path, "directory not found", ftp_code );
1775+ default:
1776+ THROW_EXCEPTION( "FTP_ERROR", path, e.what(), ftp_code );
1777+ } // switch
1778+ }
1779+ catch ( std::exception const &e ) {
1780+ THROW_EXCEPTION( "FTP_ERROR", path, e.what() );
1781+ }
1782+ return ItemSequence_t( new EmptySequence() );
1783+}
1784+
1785+///////////////////////////////////////////////////////////////////////////////
1786+
1787+} // namespace ftp_client
1788+} // namespace zorba
1789+/* vim:set et sw=2 ts=2: */
1790
1791=== added file 'modules/ftp-client/ftp-client.xq.src/ftp_functions.h'
1792--- modules/ftp-client/ftp-client.xq.src/ftp_functions.h 1970-01-01 00:00:00 +0000
1793+++ modules/ftp-client/ftp-client.xq.src/ftp_functions.h 2014-01-13 19:25:59 +0000
1794@@ -0,0 +1,159 @@
1795+/*
1796+ * Copyright 2006-2013 The FLWOR Foundation.
1797+ *
1798+ * Licensed under the Apache License, Version 2.0 (the "License");
1799+ * you may not use this file except in compliance with the License.
1800+ * You may obtain a copy of the License at
1801+ *
1802+ * http://www.apache.org/licenses/LICENSE-2.0
1803+ *
1804+ * Unless required by applicable law or agreed to in writing, software
1805+ * distributed under the License is distributed on an "AS IS" BASIS,
1806+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1807+ * See the License for the specific language governing permissions and
1808+ * limitations under the License.
1809+ */
1810+
1811+#ifndef ZORBA_FTP_CLIENT_MODULE_FUNCTIONS_H
1812+#define ZORBA_FTP_CLIENT_MODULE_FUNCTIONS_H
1813+
1814+// Zorba
1815+#include <zorba/function.h>
1816+#include <zorba/item.h>
1817+#include <zorba/zorba_string.h>
1818+
1819+#include "util/curl_streambuf.h"
1820+
1821+namespace zorba {
1822+namespace ftp_client {
1823+
1824+class module;
1825+
1826+///////////////////////////////////////////////////////////////////////////////
1827+
1828+class function : public ContextualExternalFunction {
1829+public:
1830+ // inherited
1831+ virtual String getLocalName() const;
1832+ virtual String getURI() const;
1833+
1834+protected:
1835+ function( module const *m, char const *local_name );
1836+
1837+ bool get_bool_opt( Item const&, char const*, bool = false ) const;
1838+ int get_integer_opt( Item const&, char const*, int = 0 ) const;
1839+ Item get_item_arg( ExternalFunction::Arguments_t const&, unsigned ) const;
1840+ String get_string_arg( ExternalFunction::Arguments_t const&, unsigned ) const;
1841+ String get_string_opt( Item const&, char const*, char const* = "" ) const;
1842+
1843+ curl::streambuf* require_connection( DynamicContext const*,
1844+ String const& ) const;
1845+
1846+ void throw_exception( char const*, char const*, char const*, int = 0 ) const;
1847+
1848+ void throw_exception( char const *error_code, String const &s,
1849+ char const *message, int ftp_code = 0 ) const {
1850+ throw_exception( error_code, s.c_str(), message, ftp_code );
1851+ }
1852+
1853+ void throw_exception( char const *error_code, String const &s,
1854+ String const &message, int ftp_code = 0 ) const {
1855+ throw_exception( error_code, s.c_str(), message.c_str(), ftp_code );
1856+ }
1857+
1858+ module const *const module_;
1859+ char const *const local_name_; // points to C string literal
1860+};
1861+
1862+class get_function : public function {
1863+public:
1864+ // inherited
1865+ ItemSequence_t evaluate( ExternalFunction::Arguments_t const&,
1866+ StaticContext const*, DynamicContext const* ) const;
1867+
1868+protected:
1869+ get_function( module const*, char const *local_name, bool text );
1870+
1871+ bool const text_;
1872+};
1873+
1874+class put_function : public function {
1875+public:
1876+ // inherited
1877+ ItemSequence_t evaluate( ExternalFunction::Arguments_t const&,
1878+ StaticContext const*, DynamicContext const* ) const;
1879+
1880+protected:
1881+ put_function( module const*, char const *local_name, bool text );
1882+
1883+ bool const text_;
1884+};
1885+
1886+///////////////////////////////////////////////////////////////////////////////
1887+
1888+struct connect_function : function {
1889+ connect_function( module const* );
1890+ ItemSequence_t evaluate( ExternalFunction::Arguments_t const&,
1891+ StaticContext const*, DynamicContext const* ) const;
1892+};
1893+
1894+#if 0
1895+struct disconnect_function : function {
1896+ disconnect_function( module const* );
1897+ ItemSequence_t evaluate( ExternalFunction::Arguments_t const&,
1898+ StaticContext const*, DynamicContext const* ) const;
1899+};
1900+#endif
1901+
1902+struct delete_function : function {
1903+ delete_function( module const* );
1904+ ItemSequence_t evaluate( ExternalFunction::Arguments_t const&,
1905+ StaticContext const*, DynamicContext const* ) const;
1906+};
1907+
1908+struct get_binary_function : get_function {
1909+ get_binary_function( module const* );
1910+};
1911+
1912+struct get_text_function : get_function {
1913+ get_text_function( module const* );
1914+};
1915+
1916+struct list_function : function {
1917+ list_function( module const* );
1918+ ItemSequence_t evaluate( ExternalFunction::Arguments_t const&,
1919+ StaticContext const*, DynamicContext const* ) const;
1920+};
1921+
1922+struct mkdir_function : function {
1923+ mkdir_function( module const* );
1924+ ItemSequence_t evaluate( ExternalFunction::Arguments_t const&,
1925+ StaticContext const*, DynamicContext const* ) const;
1926+};
1927+
1928+struct put_binary_function : put_function {
1929+ put_binary_function( module const* );
1930+};
1931+
1932+struct put_text_function : put_function {
1933+ put_text_function( module const* );
1934+};
1935+
1936+struct rename_function : function {
1937+ rename_function( module const* );
1938+ ItemSequence_t evaluate( ExternalFunction::Arguments_t const&,
1939+ StaticContext const*, DynamicContext const* ) const;
1940+};
1941+
1942+struct rmdir_function : function {
1943+ rmdir_function( module const* );
1944+ ItemSequence_t evaluate( ExternalFunction::Arguments_t const&,
1945+ StaticContext const*, DynamicContext const* ) const;
1946+};
1947+
1948+///////////////////////////////////////////////////////////////////////////////
1949+
1950+} // namespace ftp_client
1951+} // namespace zorba
1952+#endif /* ZORBA_FTP_CLIENT_MODULE_FUNCTIONS_H */
1953+/* vim:set et sw=2 ts=2: */
1954
1955=== added file 'modules/ftp-client/ftp-client.xq.src/ftp_module.cpp'
1956--- modules/ftp-client/ftp-client.xq.src/ftp_module.cpp 1970-01-01 00:00:00 +0000
1957+++ modules/ftp-client/ftp-client.xq.src/ftp_module.cpp 2014-01-13 19:25:59 +0000
1958@@ -0,0 +1,101 @@
1959+/*
1960+ * Copyright 2006-2013 The FLWOR Foundation.
1961+ *
1962+ * Licensed under the Apache License, Version 2.0 (the "License");
1963+ * you may not use this file except in compliance with the License.
1964+ * You may obtain a copy of the License at
1965+ *
1966+ * http://www.apache.org/licenses/LICENSE-2.0
1967+ *
1968+ * Unless required by applicable law or agreed to in writing, software
1969+ * distributed under the License is distributed on an "AS IS" BASIS,
1970+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1971+ * See the License for the specific language governing permissions and
1972+ * limitations under the License.
1973+ */
1974+
1975+// Zorba
1976+#include <zorba/zorba.h>
1977+
1978+// local
1979+#include "ftp_functions.h"
1980+#include "ftp_module.h"
1981+
1982+namespace zorba {
1983+namespace ftp_client {
1984+
1985+///////////////////////////////////////////////////////////////////////////////
1986+
1987+module::module() {
1988+ item_factory_ = 0;
1989+}
1990+
1991+module::~module() {
1992+ for ( func_map_type::const_iterator i = func_map_.begin();
1993+ i != func_map_.end(); ++i ) {
1994+ delete i->second;
1995+ }
1996+}
1997+
1998+void module::destroy() {
1999+ delete this;
2000+}
2001+
2002+ExternalFunction* module::getExternalFunction( String const &local_name ) {
2003+ ExternalFunction *&f = func_map_[ local_name ];
2004+ if ( !f ) {
2005+ if ( local_name == "connect" )
2006+ f = new connect_function( this );
2007+ else if ( local_name == "delete" )
2008+ f = new delete_function( this );
2009+#if 0
2010+ else if ( local_name == "disconnect" )
2011+ f = new disconnect_function( this );
2012+#endif
2013+ else if ( local_name == "get-binary" )
2014+ f = new get_binary_function( this );
2015+ else if ( local_name == "get-text" )
2016+ f = new get_text_function( this );
2017+ else if ( local_name == "list" )
2018+ f = new list_function( this );
2019+ else if ( local_name == "mkdir" )
2020+ f = new mkdir_function( this );
2021+ else if ( local_name == "put-binary" )
2022+ f = new put_binary_function( this );
2023+ else if ( local_name == "put-text" )
2024+ f = new put_text_function( this );
2025+ else if ( local_name == "rename" )
2026+ f = new rename_function( this );
2027+ else if ( local_name == "rmdir" )
2028+ f = new rmdir_function( this );
2029+ }
2030+ return f;
2031+}
2032+
2033+ItemFactory* module::getItemFactory() const {
2034+ if ( !item_factory_ )
2035+ item_factory_ = Zorba::getInstance(0)->getItemFactory();
2036+ return item_factory_;
2037+}
2038+
2039+String module::getURI() const {
2040+ static String const uri( "http://zorba.io/modules/ftp-client" );
2041+ return uri;
2042+}
2043+
2044+///////////////////////////////////////////////////////////////////////////////
2045+
2046+} // namespace ftp_client
2047+} // namespace zorba
2048+
2049+#ifdef WIN32
2050+# define DLL_EXPORT __declspec(dllexport)
2051+#else
2052+# define DLL_EXPORT __attribute__ ((visibility("default")))
2053+#endif /* WIN32 */
2054+
2055+extern "C" DLL_EXPORT zorba::ExternalModule* createModule() {
2056+ return new zorba::ftp_client::module();
2057+}
2058+
2059+/* vim:set et sw=2 ts=2: */
2060
2061=== added file 'modules/ftp-client/ftp-client.xq.src/ftp_module.h'
2062--- modules/ftp-client/ftp-client.xq.src/ftp_module.h 1970-01-01 00:00:00 +0000
2063+++ modules/ftp-client/ftp-client.xq.src/ftp_module.h 2014-01-13 19:25:59 +0000
2064@@ -0,0 +1,56 @@
2065+/*
2066+ * Copyright 2006-2013 The FLWOR Foundation.
2067+ *
2068+ * Licensed under the Apache License, Version 2.0 (the "License");
2069+ * you may not use this file except in compliance with the License.
2070+ * You may obtain a copy of the License at
2071+ *
2072+ * http://www.apache.org/licenses/LICENSE-2.0
2073+ *
2074+ * Unless required by applicable law or agreed to in writing, software
2075+ * distributed under the License is distributed on an "AS IS" BASIS,
2076+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
2077+ * See the License for the specific language governing permissions and
2078+ * limitations under the License.
2079+ */
2080+
2081+#ifndef ZORBA_FTP_CLIENT_FTP_MODULE_H
2082+#define ZORBA_FTP_CLIENT_FTP_MODULE_H
2083+
2084+// standard
2085+#include <map>
2086+
2087+// Zorba
2088+#include <zorba/external_module.h>
2089+
2090+namespace zorba {
2091+namespace ftp_client {
2092+
2093+///////////////////////////////////////////////////////////////////////////////
2094+
2095+class module : public ExternalModule {
2096+public:
2097+ module();
2098+ ~module();
2099+
2100+ ItemFactory* getItemFactory() const;
2101+
2102+ // inherited
2103+ virtual void destroy();
2104+ virtual ExternalFunction* getExternalFunction( String const& );
2105+ virtual String getURI() const;
2106+
2107+private:
2108+ // map function names -> ExternalFunction*
2109+ typedef std::map<String,ExternalFunction*> func_map_type;
2110+ mutable func_map_type func_map_;
2111+ mutable ItemFactory *item_factory_;
2112+};
2113+
2114+///////////////////////////////////////////////////////////////////////////////
2115+
2116+} // namespace ftp_client
2117+} // namespace zorba
2118+
2119+#endif /* ZORBA_FTP_CLIENT_FTP_MODULE_H */
2120+/* vim:set et sw=2 ts=2: */
2121
2122=== added file 'modules/ftp-client/ftp-client.xq.src/ftpparse.c'
2123--- modules/ftp-client/ftp-client.xq.src/ftpparse.c 1970-01-01 00:00:00 +0000
2124+++ modules/ftp-client/ftp-client.xq.src/ftpparse.c 2014-01-13 19:25:59 +0000
2125@@ -0,0 +1,448 @@
2126+/* ftpparse.c, ftpparse.h: library for parsing FTP LIST responses
2127+20001223
2128+D. J. Bernstein, djb@cr.yp.to
2129+http://cr.yp.to/ftpparse.html
2130+
2131+Commercial use is fine, if you let me know what programs you're using this in.
2132+
2133+Currently covered formats:
2134+EPLF.
2135+UNIX ls, with or without gid.
2136+Microsoft FTP Service.
2137+Windows NT FTP Server.
2138+VMS.
2139+WFTPD.
2140+NetPresenz (Mac).
2141+NetWare.
2142+MSDOS.
2143+
2144+Definitely not covered:
2145+Long VMS filenames, with information split across two lines.
2146+NCSA Telnet FTP server. Has LIST = NLST (and bad NLST for directories).
2147+*/
2148+
2149+#include <time.h>
2150+#include "ftpparse.h"
2151+
2152+static long totai(long year,long month,long mday)
2153+{
2154+ long result;
2155+ if (month >= 2) month -= 2;
2156+ else { month += 10; --year; }
2157+ result = (mday - 1) * 10 + 5 + 306 * month;
2158+ result /= 10;
2159+ if (result == 365) { year -= 3; result = 1460; }
2160+ else result += 365 * (year % 4);
2161+ year /= 4;
2162+ result += 1461 * (year % 25);
2163+ year /= 25;
2164+ if (result == 36524) { year -= 3; result = 146096; }
2165+ else { result += 36524 * (year % 4); }
2166+ year /= 4;
2167+ result += 146097 * (year - 5);
2168+ result += 11017;
2169+ return result * 86400;
2170+}
2171+
2172+static int flagneedbase = 1;
2173+static time_t base; /* time() value on this OS at the beginning of 1970 TAI */
2174+static long now; /* current time */
2175+static int flagneedcurrentyear = 1;
2176+static long currentyear; /* approximation to current year */
2177+
2178+static void initbase(void)
2179+{
2180+ struct tm *t;
2181+ if (!flagneedbase) return;
2182+
2183+ base = 0;
2184+ t = gmtime(&base);
2185+ base = -(totai(t->tm_year + 1900,t->tm_mon,t->tm_mday) + t->tm_hour * 3600 + t->tm_min * 60 + t->tm_sec);
2186+ /* assumes the right time_t, counting seconds. */
2187+ /* base may be slightly off if time_t counts non-leap seconds. */
2188+ flagneedbase = 0;
2189+}
2190+
2191+static void initnow(void)
2192+{
2193+ long day;
2194+ long year;
2195+
2196+ initbase();
2197+ now = time((time_t *) 0) - base;
2198+
2199+ if (flagneedcurrentyear) {
2200+ day = now / 86400;
2201+ if ((now % 86400) < 0) --day;
2202+ day -= 11017;
2203+ year = 5 + day / 146097;
2204+ day = day % 146097;
2205+ if (day < 0) { day += 146097; --year; }
2206+ year *= 4;
2207+ if (day == 146096) { year += 3; day = 36524; }
2208+ else { year += day / 36524; day %= 36524; }
2209+ year *= 25;
2210+ year += day / 1461;
2211+ day %= 1461;
2212+ year *= 4;
2213+ if (day == 1460) { year += 3; day = 365; }
2214+ else { year += day / 365; day %= 365; }
2215+ day *= 10;
2216+ if ((day + 5) / 306 >= 10) ++year;
2217+ currentyear = year;
2218+ flagneedcurrentyear = 0;
2219+ }
2220+}
2221+
2222+/* UNIX ls does not show the year for dates in the last six months. */
2223+/* So we have to guess the year. */
2224+/* Apparently NetWare uses ``twelve months'' instead of ``six months''; ugh. */
2225+/* Some versions of ls also fail to show the year for future dates. */
2226+static long guesstai(long month,long mday)
2227+{
2228+ long year;
2229+ long t;
2230+
2231+ initnow();
2232+
2233+ for (year = currentyear - 1;year < currentyear + 100;++year) {
2234+ t = totai(year,month,mday);
2235+ if (now - t < 350 * 86400)
2236+ return t;
2237+ }
2238+ return 0;
2239+}
2240+
2241+static int check(const char *buf,const char *monthname)
2242+{
2243+ if ((buf[0] != monthname[0]) && (buf[0] != monthname[0] - 32)) return 0;
2244+ if ((buf[1] != monthname[1]) && (buf[1] != monthname[1] - 32)) return 0;
2245+ if ((buf[2] != monthname[2]) && (buf[2] != monthname[2] - 32)) return 0;
2246+ return 1;
2247+}
2248+
2249+static const char *months[12] = {
2250+ "jan","feb","mar","apr","may","jun","jul","aug","sep","oct","nov","dec"
2251+} ;
2252+
2253+static int getmonth(const char *buf,int len)
2254+{
2255+ int i;
2256+ if (len == 3)
2257+ for (i = 0;i < 12;++i)
2258+ if (check(buf,months[i])) return i;
2259+ return -1;
2260+}
2261+
2262+static long getlong(const char *buf,int len)
2263+{
2264+ long u = 0;
2265+ while (len-- > 0)
2266+ u = u * 10 + (*buf++ - '0');
2267+ return u;
2268+}
2269+
2270+int ftpparse(struct ftpparse *fp,const char *buf,int len)
2271+{
2272+ int i;
2273+ int j;
2274+ int state;
2275+ long size = 0;
2276+ long year = 0;
2277+ long month = 0;
2278+ long mday = 0;
2279+ long hour = 0;
2280+ long minute = 0;
2281+
2282+ fp->name = 0;
2283+ fp->namelen = 0;
2284+ fp->flagtrycwd = 0;
2285+ fp->flagtryretr = 0;
2286+ fp->sizetype = FTPPARSE_SIZE_UNKNOWN;
2287+ fp->size = 0;
2288+ fp->mtimetype = FTPPARSE_MTIME_UNKNOWN;
2289+ fp->mtime = 0;
2290+ fp->idtype = FTPPARSE_ID_UNKNOWN;
2291+ fp->id = 0;
2292+ fp->idlen = 0;
2293+
2294+ if (len < 2) /* an empty name in EPLF, with no info, could be 2 chars */
2295+ return 0;
2296+
2297+ switch(*buf) {
2298+ /* see http://pobox.com/~djb/proto/eplf.txt */
2299+ /* "+i8388621.29609,m824255902,/,\tdev" */
2300+ /* "+i8388621.44468,m839956783,r,s10376,\tRFCEPLF" */
2301+ case '+':
2302+ i = 1;
2303+ for (j = 1;j < len;++j) {
2304+ if (buf[j] == 9) {
2305+ fp->name = buf + j + 1;
2306+ fp->namelen = len - j - 1;
2307+ return 1;
2308+ }
2309+ if (buf[j] == ',') {
2310+ switch(buf[i]) {
2311+ case '/':
2312+ fp->flagtrycwd = 1;
2313+ break;
2314+ case 'r':
2315+ fp->flagtryretr = 1;
2316+ break;
2317+ case 's':
2318+ fp->sizetype = FTPPARSE_SIZE_BINARY;
2319+ fp->size = getlong(buf + i + 1,j - i - 1);
2320+ break;
2321+ case 'm':
2322+ fp->mtimetype = FTPPARSE_MTIME_LOCAL;
2323+ initbase();
2324+ fp->mtime = base + getlong(buf + i + 1,j - i - 1);
2325+ break;
2326+ case 'i':
2327+ fp->idtype = FTPPARSE_ID_FULL;
2328+ fp->id = buf + i + 1;
2329+ fp->idlen = j - i - 1;
2330+ }
2331+ i = j + 1;
2332+ }
2333+ }
2334+ return 0;
2335+
2336+ /* UNIX-style listing, without inum and without blocks */
2337+ /* "-rw-r--r-- 1 root other 531 Jan 29 03:26 README" */
2338+ /* "dr-xr-xr-x 2 root other 512 Apr 8 1994 etc" */
2339+ /* "dr-xr-xr-x 2 root 512 Apr 8 1994 etc" */
2340+ /* "lrwxrwxrwx 1 root other 7 Jan 25 00:17 bin -> usr/bin" */
2341+ /* Also produced by Microsoft's FTP servers for Windows: */
2342+ /* "---------- 1 owner group 1803128 Jul 10 10:18 ls-lR.Z" */
2343+ /* "d--------- 1 owner group 0 May 9 19:45 Softlib" */
2344+ /* Also WFTPD for MSDOS: */
2345+ /* "-rwxrwxrwx 1 noone nogroup 322 Aug 19 1996 message.ftp" */
2346+ /* Also NetWare: */
2347+ /* "d [R----F--] supervisor 512 Jan 16 18:53 login" */
2348+ /* "- [R----F--] rhesus 214059 Oct 20 15:27 cx.exe" */
2349+ /* Also NetPresenz for the Mac: */
2350+ /* "-------r-- 326 1391972 1392298 Nov 22 1995 MegaPhone.sit" */
2351+ /* "drwxrwxr-x folder 2 May 10 1996 network" */
2352+ case 'b':
2353+ case 'c':
2354+ case 'd':
2355+ case 'l':
2356+ case 'p':
2357+ case 's':
2358+ case '-':
2359+
2360+ if (*buf == 'd') fp->flagtrycwd = 1;
2361+ if (*buf == '-') fp->flagtryretr = 1;
2362+ if (*buf == 'l') fp->flagtrycwd = fp->flagtryretr = 1;
2363+
2364+ state = 1;
2365+ i = 0;
2366+ for (j = 1;j < len;++j)
2367+ if ((buf[j] == ' ') && (buf[j - 1] != ' ')) {
2368+ switch(state) {
2369+ case 1: /* skipping perm */
2370+ state = 2;
2371+ break;
2372+ case 2: /* skipping nlink */
2373+ state = 3;
2374+ if ((j - i == 6) && (buf[i] == 'f')) /* for NetPresenz */
2375+ state = 4;
2376+ break;
2377+ case 3: /* skipping uid */
2378+ state = 4;
2379+ break;
2380+ case 4: /* getting tentative size */
2381+ size = getlong(buf + i,j - i);
2382+ state = 5;
2383+ break;
2384+ case 5: /* searching for month, otherwise getting tentative size */
2385+ month = getmonth(buf + i,j - i);
2386+ if (month >= 0)
2387+ state = 6;
2388+ else
2389+ size = getlong(buf + i,j - i);
2390+ break;
2391+ case 6: /* have size and month */
2392+ mday = getlong(buf + i,j - i);
2393+ state = 7;
2394+ break;
2395+ case 7: /* have size, month, mday */
2396+ if ((j - i == 4) && (buf[i + 1] == ':')) {
2397+ hour = getlong(buf + i,1);
2398+ minute = getlong(buf + i + 2,2);
2399+ fp->mtimetype = FTPPARSE_MTIME_REMOTEMINUTE;
2400+ initbase();
2401+ fp->mtime = base + guesstai(month,mday) + hour * 3600 + minute * 60;
2402+ } else if ((j - i == 5) && (buf[i + 2] == ':')) {
2403+ hour = getlong(buf + i,2);
2404+ minute = getlong(buf + i + 3,2);
2405+ fp->mtimetype = FTPPARSE_MTIME_REMOTEMINUTE;
2406+ initbase();
2407+ fp->mtime = base + guesstai(month,mday) + hour * 3600 + minute * 60;
2408+ }
2409+ else if (j - i >= 4) {
2410+ year = getlong(buf + i,j - i);
2411+ fp->mtimetype = FTPPARSE_MTIME_REMOTEDAY;
2412+ initbase();
2413+ fp->mtime = base + totai(year,month,mday);
2414+ }
2415+ else
2416+ return 0;
2417+ fp->name = buf + j + 1;
2418+ fp->namelen = len - j - 1;
2419+ state = 8;
2420+ break;
2421+ case 8: /* twiddling thumbs */
2422+ break;
2423+ }
2424+ i = j + 1;
2425+ while ((i < len) && (buf[i] == ' ')) ++i;
2426+ }
2427+
2428+ if (state != 8)
2429+ return 0;
2430+
2431+ fp->size = size;
2432+ fp->sizetype = FTPPARSE_SIZE_BINARY;
2433+
2434+ if (*buf == 'l')
2435+ for (i = 0;i + 3 < fp->namelen;++i)
2436+ if (fp->name[i] == ' ')
2437+ if (fp->name[i + 1] == '-')
2438+ if (fp->name[i + 2] == '>')
2439+ if (fp->name[i + 3] == ' ') {
2440+ fp->namelen = i;
2441+ break;
2442+ }
2443+
2444+ /* eliminate extra NetWare spaces */
2445+ if ((buf[1] == ' ') || (buf[1] == '['))
2446+ if (fp->namelen > 3)
2447+ if (fp->name[0] == ' ')
2448+ if (fp->name[1] == ' ')
2449+ if (fp->name[2] == ' ') {
2450+ fp->name += 3;
2451+ fp->namelen -= 3;
2452+ }
2453+
2454+ return 1;
2455+ }
2456+
2457+ /* MultiNet (some spaces removed from examples) */
2458+ /* "00README.TXT;1 2 30-DEC-1996 17:44 [SYSTEM] (RWED,RWED,RE,RE)" */
2459+ /* "CORE.DIR;1 1 8-SEP-1996 16:09 [SYSTEM] (RWE,RWE,RE,RE)" */
2460+ /* and non-MutliNet VMS: */
2461+ /* "CII-MANUAL.TEX;1 213/216 29-JAN-1996 03:33:12 [ANONYMOU,ANONYMOUS] (RWED,RWED,,)" */
2462+ for (i = 0;i < len;++i)
2463+ if (buf[i] == ';')
2464+ break;
2465+ if (i < len) {
2466+ fp->name = buf;
2467+ fp->namelen = i;
2468+ if (i > 4)
2469+ if (buf[i - 4] == '.')
2470+ if (buf[i - 3] == 'D')
2471+ if (buf[i - 2] == 'I')
2472+ if (buf[i - 1] == 'R') {
2473+ fp->namelen -= 4;
2474+ fp->flagtrycwd = 1;
2475+ }
2476+ if (!fp->flagtrycwd)
2477+ fp->flagtryretr = 1;
2478+ while (buf[i] != ' ') if (++i == len) return 0;
2479+ while (buf[i] == ' ') if (++i == len) return 0;
2480+ while (buf[i] != ' ') if (++i == len) return 0;
2481+ while (buf[i] == ' ') if (++i == len) return 0;
2482+ j = i;
2483+ while (buf[j] != '-') if (++j == len) return 0;
2484+ mday = getlong(buf + i,j - i);
2485+ while (buf[j] == '-') if (++j == len) return 0;
2486+ i = j;
2487+ while (buf[j] != '-') if (++j == len) return 0;
2488+ month = getmonth(buf + i,j - i);
2489+ if (month < 0) return 0;
2490+ while (buf[j] == '-') if (++j == len) return 0;
2491+ i = j;
2492+ while (buf[j] != ' ') if (++j == len) return 0;
2493+ year = getlong(buf + i,j - i);
2494+ while (buf[j] == ' ') if (++j == len) return 0;
2495+ i = j;
2496+ while (buf[j] != ':') if (++j == len) return 0;
2497+ hour = getlong(buf + i,j - i);
2498+ while (buf[j] == ':') if (++j == len) return 0;
2499+ i = j;
2500+ while ((buf[j] != ':') && (buf[j] != ' ')) if (++j == len) return 0;
2501+ minute = getlong(buf + i,j - i);
2502+
2503+ fp->mtimetype = FTPPARSE_MTIME_REMOTEMINUTE;
2504+ initbase();
2505+ fp->mtime = base + totai(year,month,mday) + hour * 3600 + minute * 60;
2506+
2507+ return 1;
2508+ }
2509+
2510+ /* MSDOS format */
2511+ /* 04-27-00 09:09PM <DIR> licensed */
2512+ /* 07-18-00 10:16AM <DIR> pub */
2513+ /* 04-14-00 03:47PM 589 readme.htm */
2514+ if ((*buf >= '0') && (*buf <= '9')) {
2515+ i = 0;
2516+ j = 0;
2517+ while (buf[j] != '-') if (++j == len) return 0;
2518+ month = getlong(buf + i,j - i) - 1;
2519+ while (buf[j] == '-') if (++j == len) return 0;
2520+ i = j;
2521+ while (buf[j] != '-') if (++j == len) return 0;
2522+ mday = getlong(buf + i,j - i);
2523+ while (buf[j] == '-') if (++j == len) return 0;
2524+ i = j;
2525+ while (buf[j] != ' ') if (++j == len) return 0;
2526+ year = getlong(buf + i,j - i);
2527+ if (year < 50) year += 2000;
2528+ if (year < 1000) year += 1900;
2529+ while (buf[j] == ' ') if (++j == len) return 0;
2530+ i = j;
2531+ while (buf[j] != ':') if (++j == len) return 0;
2532+ hour = getlong(buf + i,j - i);
2533+ while (buf[j] == ':') if (++j == len) return 0;
2534+ i = j;
2535+ while ((buf[j] != 'A') && (buf[j] != 'P')) if (++j == len) return 0;
2536+ minute = getlong(buf + i,j - i);
2537+ if (hour == 12) hour = 0;
2538+ if (buf[j] == 'A') if (++j == len) return 0;
2539+ if (buf[j] == 'P') { hour += 12; if (++j == len) return 0; }
2540+ if (buf[j] == 'M') if (++j == len) return 0;
2541+
2542+ while (buf[j] == ' ') if (++j == len) return 0;
2543+ if (buf[j] == '<') {
2544+ fp->flagtrycwd = 1;
2545+ while (buf[j] != ' ') if (++j == len) return 0;
2546+ }
2547+ else {
2548+ i = j;
2549+ while (buf[j] != ' ') if (++j == len) return 0;
2550+ fp->size = getlong(buf + i,j - i);
2551+ fp->sizetype = FTPPARSE_SIZE_BINARY;
2552+ fp->flagtryretr = 1;
2553+ }
2554+ while (buf[j] == ' ') if (++j == len) return 0;
2555+
2556+ fp->name = buf + j;
2557+ fp->namelen = len - j;
2558+
2559+ fp->mtimetype = FTPPARSE_MTIME_REMOTEMINUTE;
2560+ initbase();
2561+ fp->mtime = base + totai(year,month,mday) + hour * 3600 + minute * 60;
2562+
2563+ return 1;
2564+ }
2565+
2566+ /* Some useless lines, safely ignored: */
2567+ /* "Total of 11 Files, 10966 Blocks." (VMS) */
2568+ /* "total 14786" (UNIX) */
2569+ /* "DISK$ANONFTP:[ANONYMOUS]" (VMS) */
2570+ /* "Directory DISK$PCSA:[ANONYM]" (VMS) */
2571+
2572+ return 0;
2573+}
2574
2575=== added file 'modules/ftp-client/ftp-client.xq.src/ftpparse.h'
2576--- modules/ftp-client/ftp-client.xq.src/ftpparse.h 1970-01-01 00:00:00 +0000
2577+++ modules/ftp-client/ftp-client.xq.src/ftpparse.h 2014-01-13 19:25:59 +0000
2578@@ -0,0 +1,51 @@
2579+#ifndef FTPPARSE_H
2580+#define FTPPARSE_H
2581+
2582+/*
2583+ftpparse(&fp,buf,len) tries to parse one line of LIST output.
2584+
2585+The line is an array of len characters stored in buf.
2586+It should not include the terminating CR LF; so buf[len] is typically CR.
2587+
2588+If ftpparse() can't find a filename, it returns 0.
2589+
2590+If ftpparse() can find a filename, it fills in fp and returns 1.
2591+fp is a struct ftpparse, defined below.
2592+The name is an array of fp.namelen characters stored in fp.name;
2593+fp.name points somewhere within buf.
2594+*/
2595+
2596+struct ftpparse {
2597+ const char *name; /* not necessarily 0-terminated */
2598+ int namelen;
2599+ int flagtrycwd; /* 0 if cwd is definitely pointless, 1 otherwise */
2600+ int flagtryretr; /* 0 if retr is definitely pointless, 1 otherwise */
2601+ int sizetype;
2602+ long size; /* number of octets */
2603+ int mtimetype;
2604+ time_t mtime; /* modification time */
2605+ int idtype;
2606+ const char *id; /* not necessarily 0-terminated */
2607+ int idlen;
2608+} ;
2609+
2610+#define FTPPARSE_SIZE_UNKNOWN 0
2611+#define FTPPARSE_SIZE_BINARY 1 /* size is the number of octets in TYPE I */
2612+#define FTPPARSE_SIZE_ASCII 2 /* size is the number of octets in TYPE A */
2613+
2614+#define FTPPARSE_MTIME_UNKNOWN 0
2615+#define FTPPARSE_MTIME_LOCAL 1 /* time is correct */
2616+#define FTPPARSE_MTIME_REMOTEMINUTE 2 /* time zone and secs are unknown */
2617+#define FTPPARSE_MTIME_REMOTEDAY 3 /* time zone and time of day are unknown */
2618+/*
2619+When a time zone is unknown, it is assumed to be GMT. You may want
2620+to use localtime() for LOCAL times, along with an indication that the
2621+time is correct in the local time zone, and gmtime() for REMOTE* times.
2622+*/
2623+
2624+#define FTPPARSE_ID_UNKNOWN 0
2625+#define FTPPARSE_ID_FULL 1 /* unique identifier for files on this FTP server */
2626+
2627+extern int ftpparse(struct ftpparse *,const char *,int);
2628+
2629+#endif
2630
2631=== modified file 'modules/http-client/CMakeLists.txt'
2632--- modules/http-client/CMakeLists.txt 2013-07-25 14:47:04 +0000
2633+++ modules/http-client/CMakeLists.txt 2014-01-13 19:25:59 +0000
2634@@ -1,64 +1,52 @@
2635 # Copyright 2006-2012 The FLWOR Foundation.
2636-#
2637+#
2638 # Licensed under the Apache License, Version 2.0 (the "License");
2639 # you may not use this file except in compliance with the License.
2640 # You may obtain a copy of the License at
2641-#
2642+#
2643 # http://www.apache.org/licenses/LICENSE-2.0
2644-#
2645+#
2646 # Unless required by applicable law or agreed to in writing, software
2647 # distributed under the License is distributed on an "AS IS" BASIS,
2648 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
2649 # See the License for the specific language governing permissions and
2650 # limitations under the License.
2651
2652-#
2653-# cURL
2654-#
2655-SET (CURL_FOUND)
2656-IF(ZORBA_SUPPRESS_CURL)
2657- MESSAGE(STATUS "ZORBA_SUPPRESS_CURL is true - not searching for cURL library")
2658-ELSE(ZORBA_SUPPRESS_CURL)
2659- MESSAGE(STATUS "Looking for cURL")
2660- FIND_PACKAGE(CURL)
2661-
2662- IF(CURL_FOUND)
2663- MESSAGE(STATUS "Found cURL library -- " ${CURL_LIBRARIES})
2664- INCLUDE_DIRECTORIES(${CURL_INCLUDE_DIR})
2665+IF (ZORBA_HAVE_CURL)
2666+ SET(ZorbaUtilCurlModule_DIR "../util-curl")
2667+ FIND_PACKAGE(ZorbaUtilCurlModule REQUIRED)
2668+ IF (ZorbaUtilCurlModule_FOUND)
2669+ INCLUDE_DIRECTORIES("${ZorbaUtilCurlModule_INCLUDE_DIRS}")
2670
2671 DECLARE_ZORBA_MODULE(FILE xml/http-client-error.xq
2672 URI "http://expath.org/ns/error")
2673-
2674+
2675 DECLARE_ZORBA_SCHEMA(FILE xml/http-client.xsd
2676 URI "http://expath.org/ns/http-client")
2677-
2678+
2679 DECLARE_ZORBA_MODULE(FILE xml/http-client.xq VERSION 2.0
2680 URI "http://www.zorba-xquery.com/modules/http-client"
2681- LINK_LIBRARIES ${CURL_LIBRARIES})
2682-
2683+ LINK_LIBRARIES ${CURL_LIBRARIES} ${ZorbaUtilCurlModule_LIBS})
2684+
2685 DECLARE_ZORBA_MODULE(FILE json/http-client.xq VERSION 1.0
2686 URI "http://zorba.io/modules/http-client"
2687- LINK_LIBRARIES ${CURL_LIBRARIES})
2688-
2689+ LINK_LIBRARIES ${CURL_LIBRARIES} ${ZorbaUtilCurlModule_LIBS})
2690+
2691 DECLARE_ZORBA_MODULE(FILE conv/http-client-wrapper.xq VERSION 1.0
2692 URI "http://zorba.io/modules/http-client-wrapper"
2693- LINK_LIBRARIES ${CURL_LIBRARIES})
2694-
2695+ LINK_LIBRARIES ${CURL_LIBRARIES} ${ZorbaUtilCurlModule_LIBS})
2696+
2697 IF (WIN32) # Copy certificates for windows only
2698 IF (MSVC_IDE)
2699 SET(CACERT_DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/../../bin/${CMAKE_BUILD_TYPE}/cacert.pem")
2700 ELSE (MSVC_IDE)
2701- SET(CACERT_DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/../../bin/cacert.pem")
2702+ SET(CACERT_DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/../../bin/cacert.pem")
2703 ENDIF (MSVC_IDE)
2704 CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/cacert.pem" ${CACERT_DESTINATION} COPYONLY)
2705 INSTALL(FILES ${CACERT_DESTINATION} DESTINATION bin)
2706- ENDIF (WIN32)
2707-
2708- ELSE(CURL_FOUND)
2709- MESSAGE(STATUS "The cURL library was not found - http-client will not be built")
2710- ENDIF(CURL_FOUND)
2711-ENDIF(ZORBA_SUPPRESS_CURL)
2712-SET(ZORBA_HAVE_CURL ${CURL_FOUND} CACHE BOOL "Whether Zorba found cURL" FORCE)
2713-MARK_AS_ADVANCED(ZORBA_HAVE_CURL)
2714+ ENDIF (WIN32)
2715+
2716+ ENDIF (ZorbaUtilCurlModule_FOUND)
2717+ENDIF (ZORBA_HAVE_CURL)
2718
2719 # vim:set et sw=2 ts=2:
2720
2721=== modified file 'modules/http-client/json/http-client.xq'
2722--- modules/http-client/json/http-client.xq 2013-09-26 23:15:11 +0000
2723+++ modules/http-client/json/http-client.xq 2014-01-13 19:25:59 +0000
2724@@ -186,6 +186,7 @@
2725 module namespace http = "http://zorba.io/modules/http-client";
2726
2727 import module namespace libjn = "http://jsoniq.org/function-library";
2728+import module namespace util-curl = "http://zorba.io/modules/util-curl";
2729
2730 declare namespace an = "http://zorba.io/annotations";
2731 declare namespace ver = "http://zorba.io/options/versioning";
2732
2733=== removed file 'modules/http-client/json/http-client.xq.src/curl_stream_buffer.cpp'
2734--- modules/http-client/json/http-client.xq.src/curl_stream_buffer.cpp 2013-06-14 16:32:01 +0000
2735+++ modules/http-client/json/http-client.xq.src/curl_stream_buffer.cpp 1970-01-01 00:00:00 +0000
2736@@ -1,379 +0,0 @@
2737-/*
2738- * Copyright 2006-2013 The FLWOR Foundation.
2739- *
2740- * Licensed under the Apache License, Version 2.0 (the "License");
2741- * you may not use this file except in compliance with the License.
2742- * You may obtain a copy of the License at
2743- *
2744- * http://www.apache.org/licenses/LICENSE-2.0
2745- *
2746- * Unless required by applicable law or agreed to in writing, software
2747- * distributed under the License is distributed on an "AS IS" BASIS,
2748- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
2749- * See the License for the specific language governing permissions and
2750- * limitations under the License.
2751- */
2752-
2753-#include <zorba/config.h>
2754-
2755-#include <cstdlib>
2756-#include <cstring> /* for memcpy(3) */
2757-#include <iostream>
2758-#include <cassert>
2759-#ifndef WIN32
2760-#include <cerrno>
2761-#include <sys/time.h>
2762-#endif /* WIN32 */
2763-
2764-#include <curl/multi.h>
2765-
2766-#include "curl_stream_buffer.h"
2767-#include "inform_data_read.h"
2768-
2769-using namespace std;
2770-
2771-namespace zorba {
2772-namespace curl {
2773-
2774-///////////////////////////////////////////////////////////////////////////////
2775-
2776-#define ZORBA_CURL_ASSERT(expr) \
2777- do { \
2778- if ( CURLcode const code##__LINE__ = (expr) ) \
2779- throw exception( #expr, "", code##__LINE__ ); \
2780- } while (0)
2781-
2782-#define ZORBA_CURLM_ASSERT(expr) \
2783- do { \
2784- if ( CURLMcode const code##__LINE__ = (expr) ) \
2785- if ( code##__LINE__ != CURLM_CALL_MULTI_PERFORM ) \
2786- throw exception( #expr, "", code##__LINE__ ); \
2787- } while (0)
2788-
2789-exception::exception( char const *function, char const *uri, char const *msg ) :
2790- std::exception(), msg_( msg )
2791-{
2792-}
2793-
2794-exception::exception( char const *function, char const *uri, CURLcode code ) :
2795- std::exception(),
2796- msg_( curl_easy_strerror( code ) )
2797-{
2798-}
2799-
2800-exception::exception( char const *function, char const *uri, CURLMcode code ) :
2801- std::exception(),
2802- msg_( curl_multi_strerror( code ) )
2803-{
2804-}
2805-
2806-exception::~exception() throw() {
2807- // out-of-line since it's virtual
2808-}
2809-
2810-const char* exception::what() const throw() {
2811- return msg_.c_str();
2812-}
2813-
2814-///////////////////////////////////////////////////////////////////////////////
2815-
2816-CURL* create( char const *uri, write_fn_t fn, void *data ) {
2817- //
2818- // Having cURL initialization wrapped by a class and using a singleton static
2819- // instance guarantees that cURL is initialized exactly once before use and
2820- // and also is cleaned-up at program termination (when destructors for static
2821- // objects are called).
2822- //
2823- struct curl_initializer {
2824- curl_initializer() {
2825- ZORBA_CURL_ASSERT( curl_global_init( CURL_GLOBAL_ALL ) );
2826- }
2827- ~curl_initializer() {
2828- curl_global_cleanup();
2829- }
2830- };
2831- static curl_initializer initializer;
2832-
2833- CURL *const curl = curl_easy_init();
2834- if ( !curl )
2835- throw exception( "curl_easy_init()", uri, "" );
2836-
2837- try {
2838- ZORBA_CURL_ASSERT( curl_easy_setopt( curl, CURLOPT_URL, uri ) );
2839- ZORBA_CURL_ASSERT( curl_easy_setopt( curl, CURLOPT_WRITEDATA, data ) );
2840- ZORBA_CURL_ASSERT( curl_easy_setopt( curl, CURLOPT_WRITEFUNCTION, fn ) );
2841-
2842- // Tells cURL to follow redirects. CURLOPT_MAXREDIRS is by default set to -1
2843- // thus cURL will do an infinite number of redirects.
2844- ZORBA_CURL_ASSERT( curl_easy_setopt( curl, CURLOPT_FOLLOWLOCATION, 1 ) );
2845-
2846-#ifndef ZORBA_VERIFY_PEER_SSL_CERTIFICATE
2847- ZORBA_CURL_ASSERT( curl_easy_setopt( curl, CURLOPT_SSL_VERIFYPEER, 0 ) );
2848- //
2849- // CURLOPT_SSL_VERIFYHOST is left default, value 2, meaning verify that the
2850- // Common Name or Subject Alternate Name field in the certificate matches
2851- // the name of the server.
2852- //
2853- // Tested with https://www.npr.org/rss/rss.php?id=1001
2854- // About using SSL certs in curl: http://curl.haxx.se/docs/sslcerts.html
2855-#else
2856-# ifdef WIN32
2857- // set the root CA certificates file path
2858- if ( GENV.g_curl_root_CA_certificates_path[0] )
2859- ZORBA_CURL_ASSERT(
2860- curl_easy_setopt(
2861- curl, CURLOPT_CAINFO, GENV.g_curl_root_CA_certificates_path
2862- )
2863- );
2864-# endif /* WIN32 */
2865-#endif /* ZORBA_VERIFY_PEER_SSL_CERTIFICATE */
2866-
2867- //
2868- // Some servers don't like requests that are made without a user-agent
2869- // field, so we provide one.
2870- //
2871- //ZORBA_CURL_ASSERT(
2872- //curl_easy_setopt( curl, CURLOPT_USERAGENT, "libcurl-agent/1.0" )
2873- //);
2874-
2875- return curl;
2876- }
2877- catch ( ... ) {
2878- destroy( curl );
2879- throw;
2880- }
2881-}
2882-
2883-void destroy( CURL *curl ) {
2884- if ( curl ) {
2885- curl_easy_reset( curl );
2886- curl_easy_cleanup( curl );
2887- }
2888-}
2889-
2890-///////////////////////////////////////////////////////////////////////////////
2891-
2892-streambuf::streambuf() {
2893- init();
2894-}
2895-
2896-streambuf::streambuf( char const *uri ) {
2897- init();
2898- open( uri );
2899-}
2900-
2901-streambuf::streambuf( CURL *curl ) {
2902- init();
2903- curl_ = curl;
2904- ZORBA_CURL_ASSERT( curl_easy_setopt( curl, CURLOPT_WRITEDATA, this ) );
2905- ZORBA_CURL_ASSERT( curl_easy_setopt( curl, CURLOPT_WRITEFUNCTION, curl_write_callback ) );
2906- init_curlm();
2907-}
2908-
2909-streambuf::~streambuf() {
2910- free( buf_ );
2911- close();
2912-#ifdef WIN32
2913- closesocket( dummy_socket_ );
2914-#endif
2915- // If we have been assigned memory ownership of theInformer, delete it now.
2916- if ( theOwnInformer )
2917- delete theInformer;
2918-}
2919-
2920-void streambuf::close() {
2921- if ( curl_ ) {
2922- if ( curlm_ ) {
2923- curl_multi_remove_handle( curlm_, curl_ );
2924- curl_multi_cleanup( curlm_ );
2925- curlm_ = 0;
2926- }
2927- destroy( curl_ );
2928- curl_ = 0;
2929- }
2930-}
2931-
2932-void streambuf::curl_read() {
2933- buf_len_ = 0;
2934- while ( curl_running_ && !buf_len_ ) {
2935- fd_set fd_read, fd_write, fd_except;
2936- FD_ZERO( &fd_read );
2937- FD_ZERO( &fd_write );
2938- FD_ZERO( &fd_except );
2939- int max_fd = -1;
2940-#ifdef WIN32
2941- //
2942- // Windows does not like a call to select where all arguments are 0, so we
2943- // just add a dummy socket to make the call to select happy.
2944- //
2945- FD_SET( dummy_socket_, &fd_read );
2946-#endif /* WIN32 */
2947- ZORBA_CURLM_ASSERT(
2948- curl_multi_fdset( curlm_, &fd_read, &fd_write, &fd_except, &max_fd )
2949- );
2950-
2951- //
2952- // Note that the fopen.c sample code is unnecessary at best or wrong at
2953- // worst; see: http://curl.haxx.se/mail/lib-2011-05/0011.html
2954- //
2955- timeval timeout;
2956- long curl_timeout_ms;
2957- ZORBA_CURLM_ASSERT( curl_multi_timeout( curlm_, &curl_timeout_ms ) );
2958- if ( curl_timeout_ms > 0 ) {
2959- timeout.tv_sec = curl_timeout_ms / 1000;
2960- timeout.tv_usec = curl_timeout_ms % 1000 * 1000;
2961- } else {
2962- //
2963- // From curl_multi_timeout(3):
2964- //
2965- // Note: if libcurl returns a -1 timeout here, it just means that
2966- // libcurl currently has no stored timeout value. You must not wait
2967- // too long (more than a few seconds perhaps) before you call
2968- // curl_multi_perform() again.
2969- //
2970- // So we just pick some not-too-long default.
2971- //
2972- timeout.tv_sec = 1;
2973- timeout.tv_usec = 0;
2974- }
2975-
2976- switch ( select( max_fd + 1, &fd_read, &fd_write, &fd_except, &timeout ) ) {
2977- case -1: // select error
2978-#ifdef WIN32
2979- char err_buf[8];
2980- sprintf( err_buf, "%d", WSAGetLastError() );
2981- throw exception( "select()", "", err_buf );
2982-#else
2983- throw exception( "select()", "", strerror( errno ) );
2984-#endif
2985- case 0: // timeout
2986- // no break;
2987- default:
2988- CURLMcode code;
2989- do {
2990- code = curl_multi_perform( curlm_, &curl_running_ );
2991- } while ( code == CURLM_CALL_MULTI_PERFORM );
2992- ZORBA_CURLM_ASSERT( code );
2993- }
2994- }
2995- if ( theInformer )
2996- theInformer->afterRead();
2997-}
2998-
2999-size_t streambuf::curl_write_callback( void *ptr, size_t size, size_t nmemb,
3000- void *data ) {
3001- size *= nmemb;
3002- streambuf *const that = static_cast<streambuf*>( data );
3003-
3004- if ( that->theInformer )
3005- that->theInformer->beforeRead();
3006-
3007- size_t const buf_free = that->buf_capacity_ - that->buf_len_;
3008- if ( size > buf_free ) {
3009- streamoff new_capacity = that->buf_capacity_ + size - buf_free;
3010- if ( void *const new_buf =
3011- realloc( that->buf_, static_cast<size_t>( new_capacity ) ) ) {
3012- that->buf_ = static_cast<char*>( new_buf );
3013- that->buf_capacity_ = new_capacity;
3014- } else
3015- throw exception( "realloc()", "" );
3016- }
3017- ::memcpy( that->buf_ + that->buf_len_, ptr, size );
3018- that->buf_len_ += size;
3019- return size;
3020-}
3021-
3022-void streambuf::init() {
3023- buf_ = 0;
3024- buf_capacity_ = 0;
3025- buf_len_ = 0;
3026- curl_ = 0;
3027- curlm_ = 0;
3028- curl_running_ = 0;
3029- theInformer = 0;
3030- theOwnInformer = false;
3031-#ifdef WIN32
3032- dummy_socket_ = socket( AF_INET, SOCK_DGRAM, 0 );
3033- if ( dummy_socket_ == CURL_SOCKET_BAD || dummy_socket_ == INVALID_SOCKET )
3034- throw exception( "socket()", "" );
3035-#endif /* WIN32 */
3036-}
3037-
3038-void streambuf::init_curlm() {
3039- //
3040- // Lie about cURL running initially so the while-loop in curl_read() will run
3041- // at least once.
3042- //
3043- curl_running_ = 1;
3044-
3045- //
3046- // Set the "get" pointer to the end (gptr() == egptr()) so a call to
3047- // underflow() and initial data read will be triggered.
3048- //
3049- buf_len_ = buf_capacity_;
3050- setg( buf_, buf_ + buf_len_, buf_ + buf_capacity_ );
3051-
3052- //
3053- // Clean-up has to be done here with try/catch (as opposed to relying on the
3054- // destructor) because open() can be called from the constructor. If an
3055- // exception is thrown, the constructor will not have completed, hence the
3056- // object will not have been fully constructed; therefore the destructor will
3057- // not be called.
3058- //
3059- try {
3060- if ( !(curlm_ = curl_multi_init()) )
3061- throw exception( "curl_multi_init()", "" );
3062- try {
3063- ZORBA_CURLM_ASSERT( curl_multi_add_handle( curlm_, curl_ ) );
3064- }
3065- catch ( ... ) {
3066- curl_multi_cleanup( curlm_ );
3067- curlm_ = 0;
3068- throw;
3069- }
3070- }
3071- catch ( ... ) {
3072- destroy( curl_ );
3073- curl_ = 0;
3074- throw;
3075- }
3076-}
3077-
3078-int streambuf::multi_perform() {
3079- underflow();
3080- CURLMsg *msg;
3081- int msgInQueue;
3082- int error = 0;
3083- while ( (msg = curl_multi_info_read( curlm_, &msgInQueue )) ) {
3084- if ( msg->msg == CURLMSG_DONE )
3085- error = msg->data.result;
3086- }
3087- return error;
3088-}
3089-
3090-void streambuf::open( char const *uri ) {
3091- curl_ = create( uri, curl_write_callback, this );
3092-
3093- init_curlm();
3094-}
3095-
3096-streamsize streambuf::showmanyc() {
3097- return egptr() - gptr();
3098-}
3099-
3100-streambuf::int_type streambuf::underflow() {
3101- while ( true ) {
3102- if ( gptr() < egptr() )
3103- return traits_type::to_int_type( *gptr() );
3104- curl_read();
3105- if ( !buf_len_ )
3106- return traits_type::eof();
3107- setg( buf_, buf_, buf_ + buf_len_ );
3108- }
3109-}
3110-
3111-///////////////////////////////////////////////////////////////////////////////
3112-
3113-} // namespace curl
3114-} // namespace zorba
3115-/* vim:set et sw=2 ts=2: */
3116
3117=== removed file 'modules/http-client/json/http-client.xq.src/curl_stream_buffer.h'
3118--- modules/http-client/json/http-client.xq.src/curl_stream_buffer.h 2013-06-14 16:32:01 +0000
3119+++ modules/http-client/json/http-client.xq.src/curl_stream_buffer.h 1970-01-01 00:00:00 +0000
3120@@ -1,193 +0,0 @@
3121-/*
3122- * Copyright 2006-2013 The FLWOR Foundation.
3123- *
3124- * Licensed under the Apache License, Version 2.0 (the "License");
3125- * you may not use this file except in compliance with the License.
3126- * You may obtain a copy of the License at
3127- *
3128- * http://www.apache.org/licenses/LICENSE-2.0
3129- *
3130- * Unless required by applicable law or agreed to in writing, software
3131- * distributed under the License is distributed on an "AS IS" BASIS,
3132- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
3133- * See the License for the specific language governing permissions and
3134- * limitations under the License.
3135- */
3136-#pragma once
3137-#ifndef ZORBA_CURL_UTIL_H
3138-#define ZORBA_CURL_UTIL_H
3139-
3140-#include <zorba/config.h>
3141-
3142-#include <exception>
3143-#include <istream>
3144-#include <streambuf>
3145-#include <string>
3146-#include <curl/curl.h>
3147-
3148-namespace zorba {
3149-
3150-namespace http_client {
3151- class InformDataRead;
3152-}
3153-
3154-namespace curl {
3155-
3156-///////////////////////////////////////////////////////////////////////////////
3157-
3158-class exception : public std::exception {
3159-public:
3160- exception( char const *function, char const *uri, char const *msg = 0 );
3161- exception( char const *function, char const *uri, CURLcode code );
3162- exception( char const *function, char const *uri, CURLMcode code );
3163- ~exception() throw();
3164-
3165- virtual const char* what() const throw();
3166-
3167-private:
3168- std::string msg_;
3169-};
3170-
3171-////////// create & destroy ///////////////////////////////////////////////////
3172-
3173-/**
3174- * The signature type of cURL's write function callback.
3175- */
3176-typedef size_t (*write_fn_t)( void*, size_t, size_t, void* );
3177-
3178-/**
3179- * Creates a new, initialized cURL instance.
3180- *
3181- * @throws exception upon failure.
3182- */
3183-CURL* create( char const *uri, write_fn_t fn, void *data );
3184-
3185-/**
3186- * Destroys a cURL instance.
3187- *
3188- * @param instance A cURL instance. If \c NULL, does nothing.
3189- */
3190-void destroy( CURL *instance );
3191-
3192-////////// streambuf //////////////////////////////////////////////////////////
3193-
3194-/**
3195- * A curl::streambuf is-a std::streambuf for streaming the contents of URI
3196- * using cURL. However, do not use this class directly. Use uri::streambuf
3197- * instead.
3198- */
3199-class streambuf : public std::streambuf {
3200-public:
3201- /**
3202- * Constructs a %streambuf.
3203- */
3204- streambuf();
3205-
3206- /**
3207- * Constructs a %streambuf and opens a connection to the server hosting the
3208- * given URI for subsequent streaming.
3209- *
3210- * @param uri The URI to stream.
3211- */
3212- streambuf( char const *uri );
3213-
3214- /**
3215- * Constructs a %streambuf using an existing CURL object.
3216- *
3217- * @param curl The CURL object to use. This %streambuf takes ownership of
3218- * it.
3219- */
3220- streambuf( CURL *curl );
3221-
3222- /**
3223- * Destroys a %streambuf.
3224- */
3225- ~streambuf();
3226-
3227- /**
3228- * Opens a connection to the server hosting the given URI for subsequent
3229- * streaming.
3230- *
3231- * @param uri The URI to stream.
3232- * @throws exception upon failure.
3233- */
3234- void open( char const *uri );
3235-
3236- /**
3237- * Tests whether the buffer is open.
3238- *
3239- * @return Returns \c true only if the buffer is open.
3240- */
3241- bool is_open() const {
3242- return !!curl_;
3243- }
3244-
3245- /**
3246- * Closes this %streambuf.
3247- */
3248- void close();
3249-
3250- /**
3251- * Gets the CURL object in use.
3252- *
3253- * @return Return said CURL object.
3254- */
3255- CURL* curl() const {
3256- return curl_;
3257- }
3258-
3259- /**
3260- * Provide a InformDataRead that will get callbacks about read events.
3261- */
3262- void setInformer( http_client::InformDataRead *aInformer ) {
3263- theInformer = aInformer;
3264- }
3265-
3266- /**
3267- * Specify whether this streambuf has memory ownership over the
3268- * InformDataRead it has been passed. You can use this if, for example,
3269- * the lifetime of the streambuf will extend past the lifetime of the
3270- * object which created the InformDataRead.
3271- */
3272- void setOwnInformer( bool aOwnInformer ) {
3273- theOwnInformer = aOwnInformer;
3274- }
3275-
3276- int multi_perform();
3277-
3278-protected:
3279- // inherited
3280- std::streamsize showmanyc();
3281- int_type underflow();
3282-
3283-private:
3284- void curl_read();
3285- static size_t curl_write_callback( void*, size_t, size_t, void* );
3286-
3287- void init();
3288- void init_curlm();
3289-
3290- char *buf_;
3291- std::streamsize buf_capacity_;
3292- std::streamoff buf_len_;
3293-
3294- CURL *curl_;
3295- CURLM *curlm_;
3296- int curl_running_;
3297- http_client::InformDataRead *theInformer;
3298- bool theOwnInformer;
3299-
3300- // forbid
3301- streambuf( streambuf const& );
3302- streambuf& operator=( streambuf const& );
3303-#ifdef WIN32
3304- SOCKET dummy_socket_;
3305-#endif /* WIN32 */
3306-};
3307-
3308-///////////////////////////////////////////////////////////////////////////////
3309-
3310-} // namespace curl
3311-} // namespace zorba
3312-#endif /* ZORBA_CURL_UTIL_H */
3313-/* vim:set et sw=2 ts=2: */
3314
3315=== modified file 'modules/http-client/json/http-client.xq.src/http_client.cpp'
3316--- modules/http-client/json/http-client.xq.src/http_client.cpp 2014-01-10 01:37:44 +0000
3317+++ modules/http-client/json/http-client.xq.src/http_client.cpp 2014-01-13 19:25:59 +0000
3318@@ -166,7 +166,7 @@
3319 ItemFactory* aFactory,
3320 const String& aTheModuleURI)
3321 {
3322- CURL* lCURL = curl_easy_init();
3323+ CURL* lCURL = curl::create();
3324
3325 Item lRequest;
3326 Item lHref;
3327
3328=== modified file 'modules/http-client/json/http-client.xq.src/http_response_parser.cpp'
3329--- modules/http-client/json/http-client.xq.src/http_response_parser.cpp 2013-09-23 09:11:02 +0000
3330+++ modules/http-client/json/http-client.xq.src/http_response_parser.cpp 2014-01-13 19:25:59 +0000
3331@@ -36,9 +36,10 @@
3332 #include <zorba/xquery_functions.h>
3333 #include <zorba/internal/unique_ptr.h>
3334
3335+#include "util/curl_streambuf.h"
3336+
3337 #include "http_response_parser.h"
3338 #include "http_request_handler.h"
3339-#include "curl_stream_buffer.h"
3340
3341 namespace zorba {
3342
3343@@ -94,10 +95,10 @@
3344 if (!t.empty())
3345 {
3346 if (t[0] == '"' && t[t.length()-1] == '"')
3347- {
3348+ {
3349 t.erase( 0, 1 );
3350- t.erase(t.length() -1, 1);
3351- }
3352+ t.erase(t.length() -1, 1);
3353+ }
3354 *charset = t;
3355 }
3356 }
3357@@ -129,11 +130,11 @@
3358
3359 int HttpResponseParser::parse()
3360 {
3361- theStreamBuffer->setInformer(this);
3362+ theStreamBuffer->set_listener(this);
3363 theHandler.begin();
3364 bool lStatusAndMesssageParsed = false;
3365 int lCode = 0;
3366- lCode = theStreamBuffer->multi_perform();
3367+ lCode = theStreamBuffer->curl_multi_info_read(false);
3368 if (lCode)
3369 return lCode;
3370 if (!theStatusOnly) {
3371@@ -203,7 +204,7 @@
3372 return lCode;
3373 }
3374
3375- void HttpResponseParser::beforeRead()
3376+ void HttpResponseParser::curl_read(void*,size_t)
3377 {
3378 if (theInsideRead) {
3379 return;
3380@@ -218,10 +219,6 @@
3381 theHandler.beginBody(theCurrentContentType, "", NULL);
3382 }
3383
3384- void HttpResponseParser::afterRead()
3385- {
3386- }
3387-
3388 void HttpResponseParser::registerHandler()
3389 {
3390 curl_easy_setopt(theCurl, CURLOPT_HEADERFUNCTION, &curl_headerfunction);
3391@@ -343,7 +340,7 @@
3392 // theStreamBuffer. Therefore, this HttpResponseParser object is no longer
3393 // "self-contained". We delegate ownership of ourself to theStreamBuffer
3394 // and mark ourselves as no longer being self-contained.
3395- theStreamBuffer->setOwnInformer(true);
3396+ theStreamBuffer->set_listener(this, true);
3397 theSelfContained = false;
3398
3399 // The ownership of theStreamBuffer, in turn, is delegated to the
3400
3401=== modified file 'modules/http-client/json/http-client.xq.src/http_response_parser.h'
3402--- modules/http-client/json/http-client.xq.src/http_response_parser.h 2014-01-10 01:37:44 +0000
3403+++ modules/http-client/json/http-client.xq.src/http_response_parser.h 2014-01-13 19:25:59 +0000
3404@@ -19,26 +19,21 @@
3405 #include <string>
3406 #include <map>
3407
3408-#include <curl/curl.h>
3409+#include "util/curl_streambuf.h"
3410
3411-#include "inform_data_read.h"
3412 #include "error_thrower.h"
3413 #include "http_response_handler.h"
3414
3415 namespace zorba {
3416 class Item;
3417
3418-namespace curl {
3419- class streambuf;
3420-}
3421-
3422 namespace http_client
3423 {
3424 void parse_content_type( std::string const &s, std::string *mime_type, std::string *charset );
3425
3426 class HttpResponseHandler;
3427
3428- class HttpResponseParser : public InformDataRead {
3429+ class HttpResponseParser : public curl::listener {
3430 private:
3431 HttpResponseHandler& theHandler;
3432 CURL* theCurl;
3433@@ -49,7 +44,7 @@
3434 headers_type theHeaders;
3435 int theStatus;
3436 std::string theMessage;
3437- zorba::curl::streambuf* theStreamBuffer;
3438+ curl::streambuf* theStreamBuffer;
3439 std::string theId;
3440 std::string theDescription;
3441 bool theInsideRead;
3442@@ -76,8 +71,7 @@
3443 * will return false.
3444 */
3445 bool selfContained() { return theSelfContained; }
3446- virtual void beforeRead();
3447- virtual void afterRead();
3448+ virtual void curl_read(void*,size_t);
3449 private:
3450 void registerHandler();
3451 void parseStatusAndMessage(std::string const &aHeader);
3452
3453=== removed file 'modules/http-client/json/http-client.xq.src/inform_data_read.cpp'
3454--- modules/http-client/json/http-client.xq.src/inform_data_read.cpp 2014-01-10 01:37:44 +0000
3455+++ modules/http-client/json/http-client.xq.src/inform_data_read.cpp 1970-01-01 00:00:00 +0000
3456@@ -1,23 +0,0 @@
3457-/*
3458- * Copyright 2006-2013 The FLWOR Foundation.
3459- *
3460- * Licensed under the Apache License, Version 2.0 (the "License");
3461- * you may not use this file except in compliance with the License.
3462- * You may obtain a copy of the License at
3463- *
3464- * http://www.apache.org/licenses/LICENSE-2.0
3465- *
3466- * Unless required by applicable law or agreed to in writing, software
3467- * distributed under the License is distributed on an "AS IS" BASIS,
3468- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
3469- * See the License for the specific language governing permissions and
3470- * limitations under the License.
3471- */
3472-#include "inform_data_read.h"
3473-
3474-namespace zorba { namespace http_client {
3475- InformDataRead::~InformDataRead()
3476- {
3477- }
3478-}} //namespace zorba, http_client
3479-/* vim:set et sw=2 ts=2: */
3480
3481=== removed file 'modules/http-client/json/http-client.xq.src/inform_data_read.h'
3482--- modules/http-client/json/http-client.xq.src/inform_data_read.h 2014-01-10 01:37:44 +0000
3483+++ modules/http-client/json/http-client.xq.src/inform_data_read.h 1970-01-01 00:00:00 +0000
3484@@ -1,28 +0,0 @@
3485-/*
3486- * Copyright 2006-2013 The FLWOR Foundation.
3487- *
3488- * Licensed under the Apache License, Version 2.0 (the "License");
3489- * you may not use this file except in compliance with the License.
3490- * You may obtain a copy of the License at
3491- *
3492- * http://www.apache.org/licenses/LICENSE-2.0
3493- *
3494- * Unless required by applicable law or agreed to in writing, software
3495- * distributed under the License is distributed on an "AS IS" BASIS,
3496- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
3497- * See the License for the specific language governing permissions and
3498- * limitations under the License.
3499- */
3500-#ifndef INFORM_DATA_READ_H
3501-#define INFORM_DATA_READ_H
3502-namespace zorba { namespace http_client {
3503- class InformDataRead {
3504- public:
3505- virtual ~InformDataRead();
3506- public:
3507- virtual void beforeRead() = 0;
3508- virtual void afterRead() = 0;
3509- };
3510-}} //namespace zorba, http_client
3511-#endif //INFORM_DATA_READ_H
3512-/* vim:set et sw=2 ts=2: */
3513
3514=== added directory 'modules/util-curl'
3515=== added file 'modules/util-curl/CMakeLists.txt'
3516--- modules/util-curl/CMakeLists.txt 1970-01-01 00:00:00 +0000
3517+++ modules/util-curl/CMakeLists.txt 2014-01-13 19:25:59 +0000
3518@@ -0,0 +1,50 @@
3519+# Copyright 2006-2012 The FLWOR Foundation.
3520+#
3521+# Licensed under the Apache License, Version 2.0 (the "License");
3522+# you may not use this file except in compliance with the License.
3523+# You may obtain a copy of the License at
3524+#
3525+# http://www.apache.org/licenses/LICENSE-2.0
3526+#
3527+# Unless required by applicable law or agreed to in writing, software
3528+# distributed under the License is distributed on an "AS IS" BASIS,
3529+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
3530+# See the License for the specific language governing permissions and
3531+# limitations under the License.
3532+
3533+PROJECT(ZorbaUtilCurlModule)
3534+
3535+IF (ZORBA_SUPPRESS_CURL)
3536+ MESSAGE(STATUS "ZORBA_SUPPRESS_CURL is true - not searching for cURL library")
3537+ELSE (ZORBA_SUPPRESS_CURL)
3538+ MESSAGE(STATUS "Looking for cURL")
3539+ FIND_PACKAGE(CURL)
3540+
3541+ IF (CURL_FOUND)
3542+ MESSAGE(STATUS "Found cURL library -- " ${CURL_LIBRARIES})
3543+ INCLUDE_DIRECTORIES(BEFORE SYSTEM "${CURL_INCLUDE_DIR}")
3544+ SET(requiredlibs ${requiredlibs} "${CURL_LIBRARIES}")
3545+
3546+ SET(ZORBA_PROJECT_INCLUDE_DIRS "${CMAKE_CURRENT_SOURCE_DIR}/include")
3547+ INCLUDE_DIRECTORIES("${CMAKE_CURRENT_SOURCE_DIR}/include")
3548+ INSTALL(FILES include/util/curl_streambuf.h DESTINATION include/util)
3549+
3550+ ADD_SUBDIRECTORY("src")
3551+
3552+ IF (WIN32) # Copy certificates for windows only
3553+ IF (MSVC_IDE)
3554+ SET(CACERT_DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/../../bin/${CMAKE_BUILD_TYPE}/cacert.pem")
3555+ ELSE (MSVC_IDE)
3556+ SET(CACERT_DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/../../bin/cacert.pem")
3557+ ENDIF (MSVC_IDE)
3558+ CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/cacert.pem" ${CACERT_DESTINATION} COPYONLY)
3559+ INSTALL(FILES ${CACERT_DESTINATION} DESTINATION bin)
3560+ ENDIF (WIN32)
3561+
3562+ ELSE (CURL_FOUND)
3563+ MESSAGE(STATUS "The cURL library was not found")
3564+ ENDIF (CURL_FOUND)
3565+ENDIF (ZORBA_SUPPRESS_CURL)
3566+SET(ZORBA_HAVE_CURL ${CURL_FOUND} CACHE BOOL "Whether Zorba found cURL" FORCE)
3567+
3568+# vim:set et sw=2 ts=2:
3569
3570=== added file 'modules/util-curl/ZorbaUtilCurlModuleConfig.cmake'
3571--- modules/util-curl/ZorbaUtilCurlModuleConfig.cmake 1970-01-01 00:00:00 +0000
3572+++ modules/util-curl/ZorbaUtilCurlModuleConfig.cmake 2014-01-13 19:25:59 +0000
3573@@ -0,0 +1,19 @@
3574+# Copyright 2006-2013 The FLWOR Foundation.
3575+#
3576+# Licensed under the Apache License, Version 2.0 (the "License");
3577+# you may not use this file except in compliance with the License.
3578+# You may obtain a copy of the License at
3579+#
3580+# http://www.apache.org/licenses/LICENSE-2.0
3581+#
3582+# Unless required by applicable law or agreed to in writing, software
3583+# distributed under the License is distributed on an "AS IS" BASIS,
3584+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
3585+# See the License for the specific language governing permissions and
3586+# limitations under the License.
3587+
3588+SET(ZorbaUtilCurlModule_FOUND TRUE)
3589+SET(ZorbaUtilCurlModule_INCLUDES "../util-curl/include")
3590+SET(ZorbaUtilCurlModule_INCLUDE_DIRS "${ZorbaUtilCurlModule_INCLUDES}")
3591+SET(ZorbaUtilCurlModule_LIBS util-curl)
3592+SET(ZorbaUtilCurlModule_LIBRARIES "${ZorbaUtilCurlModule_LIBS}")
3593
3594=== added directory 'modules/util-curl/include'
3595=== added directory 'modules/util-curl/include/util'
3596=== added file 'modules/util-curl/include/util/curl_streambuf.h'
3597--- modules/util-curl/include/util/curl_streambuf.h 1970-01-01 00:00:00 +0000
3598+++ modules/util-curl/include/util/curl_streambuf.h 2014-01-13 19:25:59 +0000
3599@@ -0,0 +1,316 @@
3600+/*
3601+ * Copyright 2006-2013 The FLWOR Foundation.
3602+ *
3603+ * Licensed under the Apache License, Version 2.0 (the "License");
3604+ * you may not use this file except in compliance with the License.
3605+ * You may obtain a copy of the License at
3606+ *
3607+ * http://www.apache.org/licenses/LICENSE-2.0
3608+ *
3609+ * Unless required by applicable law or agreed to in writing, software
3610+ * distributed under the License is distributed on an "AS IS" BASIS,
3611+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
3612+ * See the License for the specific language governing permissions and
3613+ * limitations under the License.
3614+ */
3615+#pragma once
3616+#ifndef ZORBA_CURL_STREAMBUF_API_H
3617+#define ZORBA_CURL_STREAMBUF_API_H
3618+
3619+// standard
3620+#include <cstdlib>
3621+#include <exception>
3622+#include <streambuf>
3623+#include <string>
3624+#ifdef WIN32
3625+#include <windows.h>
3626+#endif /* WIN32 */
3627+
3628+// libcurl
3629+#include <curl/curl.h>
3630+
3631+// Zorba
3632+#include <zorba/config.h>
3633+#include <zorba/util/fs_util.h>
3634+
3635+namespace zorba {
3636+namespace curl {
3637+
3638+////////// exception //////////////////////////////////////////////////////////
3639+
3640+/**
3641+ * A curl::exception is-an exception for cURL errors. These are thrown instead
3642+ * of simply returning error codes (that are often ignored).
3643+ */
3644+class ZORBA_DLL_PUBLIC exception : public std::exception {
3645+public:
3646+ exception( char const *function, char const *uri, char const *msg = 0 );
3647+ exception( char const *function, char const *uri, CURLcode code );
3648+ exception( char const *function, char const *uri, CURLMcode code );
3649+ ~exception() throw();
3650+
3651+ /**
3652+ * Gets the cURL error code (returned from the "easy" interface).
3653+ *
3654+ * @return Returns said error code or 0 if none.
3655+ */
3656+ CURLcode curl_code() const {
3657+ return curl_code_;
3658+ }
3659+
3660+ /**
3661+ * Gets the cURL error code (returned from the "multi" interface).
3662+ *
3663+ * @return Returns said error code or 0 if none.
3664+ */
3665+ CURLMcode curlm_code() const {
3666+ return curlm_code_;
3667+ }
3668+
3669+ // inherited
3670+ virtual const char* what() const throw();
3671+
3672+private:
3673+ CURLcode curl_code_;
3674+ CURLMcode curlm_code_;
3675+ std::string msg_;
3676+};
3677+
3678+/**
3679+ * If defined, all calls to cURL wrapped by either ZORBA_CURL_ASSERT() or
3680+ * ZORBA_CURLM_ASSERT() will be printed to standard error for debugging.
3681+ */
3682+//#define ZORBA_TRACE_LIBCURL_CALLS 1
3683+
3684+#ifdef ZORBA_TRACE_LIBCURL_CALLS
3685+# define ZORBA_CURL_ECHO(CURL_FN) \
3686+ std::cerr << zorba::fs::base_name( __FILE__ ) << ':' << __LINE__ << ": " << #CURL_FN << std::endl
3687+#else
3688+# define ZORBA_CURL_ECHO(CURL_FN) (void)0
3689+#endif /* ZORBA_TRACE_LIBCURL_CALLS */
3690+
3691+/**
3692+ * Asserts that a call to a \c curl_easy_*() function
3693+ * returning a \c CURLcode succeeds; if not, throws an exception.
3694+ *
3695+ * @param EXPR A \c curl_easy_*() function call.
3696+ * \hideinitializer
3697+ */
3698+#define ZORBA_CURL_ASSERT(EXPR) \
3699+ do { \
3700+ ZORBA_CURL_ECHO( EXPR ); \
3701+ if ( CURLcode const code##__LINE__ = (EXPR) ) \
3702+ throw zorba::curl::exception( #EXPR, "", code##__LINE__ ); \
3703+ } while (0)
3704+
3705+/**
3706+ * Asserts that a call to a \c curl_multi_*() function
3707+ * returning a \c CURLMcode succeeds; if not, throws an exception.
3708+ *
3709+ * @param EXPR A \c curl_multi_*() function call.
3710+ * \hideinitializer
3711+ */
3712+#define ZORBA_CURLM_ASSERT(EXPR) \
3713+ do { \
3714+ ZORBA_CURL_ECHO( EXPR ); \
3715+ if ( CURLMcode const code##__LINE__ = (EXPR) ) \
3716+ if ( code##__LINE__ != CURLM_CALL_MULTI_PERFORM ) \
3717+ throw zorba::curl::exception( #EXPR, "", code##__LINE__ ); \
3718+ } while (0)
3719+
3720+////////// streambuf //////////////////////////////////////////////////////////
3721+
3722+/**
3723+ * A listener can be used to "listen" to the raw data that cURL reads at the
3724+ * time it reads it.
3725+ */
3726+struct ZORBA_DLL_PUBLIC listener {
3727+ virtual ~listener();
3728+
3729+ /**
3730+ * This is called whenever cURL reads data and gives it to the streambuf.
3731+ *
3732+ * @param data A pointer to the data read.
3733+ * @param size The number of bytes read.
3734+ */
3735+ virtual void curl_read( void *data, size_t size ) = 0;
3736+};
3737+
3738+////////// create/destroy /////////////////////////////////////////////////////
3739+
3740+/**
3741+ * Creates a new, initialized cURL instance.
3742+ *
3743+ * @return Returns said instance.
3744+ * @throws exception upon failure.
3745+ */
3746+ZORBA_DLL_PUBLIC
3747+CURL* create();
3748+
3749+/**
3750+ * Destroys a cURL instance.
3751+ *
3752+ * @param instance A cURL instance. If \c NULL, does nothing.
3753+ */
3754+ZORBA_DLL_PUBLIC
3755+void destroy( CURL *instance );
3756+
3757+////////// streambuf //////////////////////////////////////////////////////////
3758+
3759+/**
3760+ * A curl::streambuf is-a std::streambuf for streaming the contents of URI
3761+ * using cURL. Note that this streambuf can be used only for reading
3762+ * (downloading) and not writing (uploading) via, say, FTP or HTTP.
3763+ */
3764+class ZORBA_DLL_PUBLIC streambuf : public std::streambuf {
3765+public:
3766+ /**
3767+ * Constructs a %streambuf.
3768+ */
3769+ streambuf();
3770+
3771+ /**
3772+ * Constructs a %streambuf and opens a connection to the server hosting the
3773+ * given URI for subsequent streaming.
3774+ *
3775+ * @param uri The URI to stream.
3776+ */
3777+ streambuf( char const *uri );
3778+
3779+ /**
3780+ * Constructs a %streambuf using an existing CURL object.
3781+ *
3782+ * @param curl The CURL object to use. This %streambuf takes ownership of
3783+ * it.
3784+ */
3785+ streambuf( CURL *curl );
3786+
3787+ /**
3788+ * Destroys a %streambuf.
3789+ */
3790+ ~streambuf();
3791+
3792+ /**
3793+ * Opens a connection to the server hosting the given URI for subsequent
3794+ * streaming.
3795+ *
3796+ * @param uri The URI to stream.
3797+ * @throws exception upon failure.
3798+ */
3799+ void open( char const *uri );
3800+
3801+ /**
3802+ * Tests whether the buffer is open.
3803+ *
3804+ * @return Returns \c true only if the buffer is open.
3805+ */
3806+ bool is_open() const {
3807+ return !!curl_;
3808+ }
3809+
3810+ /**
3811+ * Closes this %streambuf.
3812+ */
3813+ void close();
3814+
3815+ /**
3816+ * Gets the CURL ("easy") object in use.
3817+ *
3818+ * @return Return said CURL object.
3819+ */
3820+ CURL* curl() const {
3821+ return curl_;
3822+ }
3823+
3824+ /**
3825+ * Gets the CURLM ("multi") object in use.
3826+ *
3827+ * @return Return said CURLM object.
3828+ */
3829+ CURLM* curlm() const {
3830+ return curlm_;
3831+ }
3832+
3833+ /**
3834+ * Resets all options on the CURL object in use.
3835+ */
3836+ void curl_reset() {
3837+ curl_init();
3838+ }
3839+
3840+ /**
3841+ * Sets/clears CURL verbose mode.
3842+ *
3843+ * @param verbose If \c true, sets verbose mode; otherwise clears it.
3844+ */
3845+ void curl_verbose( bool verbose );
3846+
3847+ /**
3848+ * Sets the listener, if any.
3849+ * If this streambuf currently has a listener that it has taken ownership of
3850+ * and \a new_listener is different from the current listener, then the
3851+ * current listner is destroyed first.
3852+ *
3853+ * @param The new_listener The listener to use. May be \c NULL.
3854+ * @param take_ownership If \c true, takes ownership of \a new_listener:
3855+ * when this streambuf is destroyed, the listener will be also.
3856+ */
3857+ void set_listener( listener *new_listener, bool take_ownership = false );
3858+
3859+public:
3860+ /**
3861+ * Reads data.
3862+ *
3863+ * @param throw_on_error If \c true and an error occurs, throws an exception.
3864+ * \deprecated This function was added and is kept only because it's used by
3865+ * the http-client module. Do not use this function in new code.
3866+ */
3867+ CURLcode curl_multi_info_read( bool throw_on_error = true );
3868+
3869+protected:
3870+ // inherited
3871+ std::streamsize showmanyc();
3872+ int_type underflow();
3873+ std::streamsize xsgetn( char_type*, std::streamsize );
3874+
3875+private:
3876+ void curl_destroy() {
3877+ destroy( curl_ );
3878+ curl_ = 0;
3879+ }
3880+ void curl_init();
3881+ void curlm_init();
3882+ void curl_io( size_t* );
3883+ void curl_write();
3884+ static size_t curl_write_callback( void*, size_t, size_t, void* );
3885+ void init();
3886+
3887+ struct gbuf {
3888+ char *ptr_;
3889+ size_t capacity_, len_;
3890+ gbuf() : ptr_( 0 ), capacity_( 0 ), len_( 0 ) { }
3891+ ~gbuf() { free( ptr_ ); }
3892+ };
3893+
3894+ CURL *curl_;
3895+ CURLM *curlm_;
3896+ int curl_running_;
3897+ gbuf gbuf_;
3898+ listener *listener_;
3899+ bool listener_owner_;
3900+ bool verbose_;
3901+
3902+ // forbid
3903+ streambuf( streambuf const& );
3904+ streambuf& operator=( streambuf const& );
3905+#ifdef WIN32
3906+ SOCKET dummy_socket_;
3907+#endif /* WIN32 */
3908+};
3909+
3910+///////////////////////////////////////////////////////////////////////////////
3911+
3912+} // namespace curl
3913+} // namespace zorba
3914+#endif /* ZORBA_CURL_STREAMBUF_API_H */
3915+/* vim:set et sw=2 ts=2: */
3916
3917=== added directory 'modules/util-curl/src'
3918=== added file 'modules/util-curl/src/CMakeLists.txt'
3919--- modules/util-curl/src/CMakeLists.txt 1970-01-01 00:00:00 +0000
3920+++ modules/util-curl/src/CMakeLists.txt 2014-01-13 19:25:59 +0000
3921@@ -0,0 +1,36 @@
3922+# Copyright 2006-2010 The FLWOR Foundation.
3923+#
3924+# Licensed under the Apache License, Version 2.0 (the "License");
3925+# you may not use this file except in compliance with the License.
3926+# You may obtain a copy of the License at
3927+#
3928+# http://www.apache.org/licenses/LICENSE-2.0
3929+#
3930+# Unless required by applicable law or agreed to in writing, software
3931+# distributed under the License is distributed on an "AS IS" BASIS,
3932+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
3933+# See the License for the specific language governing permissions and
3934+# limitations under the License.
3935+
3936+DECLARE_ZORBA_MODULE(FILE "util-curl.xq" VERSION 1.0
3937+ URI "http://zorba.io/modules/util-curl")
3938+
3939+# The important stuff is the library that we install in Zorba's default lib
3940+# directory.
3941+ADD_LIBRARY(util-curl SHARED curl_streambuf.cpp util-curl.cpp)
3942+TARGET_LINK_LIBRARIES(util-curl zorba_${ZORBA_STORE_NAME} ${CURL_LIBRARIES})
3943+INSTALL(TARGETS util-curl
3944+ RUNTIME DESTINATION bin
3945+ LIBRARY DESTINATION lib${LIB_SUFFIX}
3946+ ARCHIVE DESTINATION lib)
3947+
3948+# MAC OS X only property
3949+# This is required to make sure that the library has the correct install name
3950+# after installation such that dependent modules always find it.
3951+SET_TARGET_PROPERTIES(util-curl PROPERTIES INSTALL_NAME_DIR "${CMAKE_INSTALL_PREFIX}/lib")
3952+
3953+# Set this in the parent scope so it will be put into our Config.cmake file.
3954+# As with many other things, this doesn't work when the module is installed.
3955+SET(ZORBA_PROJECT_LIBRARIES util-curl PARENT_SCOPE)
3956+
3957+# vim:set et sw=2 ts=2:
3958
3959=== added file 'modules/util-curl/src/curl_streambuf.cpp'
3960--- modules/util-curl/src/curl_streambuf.cpp 1970-01-01 00:00:00 +0000
3961+++ modules/util-curl/src/curl_streambuf.cpp 2014-01-13 19:25:59 +0000
3962@@ -0,0 +1,389 @@
3963+/*
3964+ * Copyright 2006-2013 The FLWOR Foundation.
3965+ *
3966+ * Licensed under the Apache License, Version 2.0 (the "License");
3967+ * you may not use this file except in compliance with the License.
3968+ * You may obtain a copy of the License at
3969+ *
3970+ * http://www.apache.org/licenses/LICENSE-2.0
3971+ *
3972+ * Unless required by applicable law or agreed to in writing, software
3973+ * distributed under the License is distributed on an "AS IS" BASIS,
3974+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
3975+ * See the License for the specific language governing permissions and
3976+ * limitations under the License.
3977+ */
3978+
3979+// standard
3980+#include <algorithm>
3981+#include <cstdlib>
3982+#include <cstring> /* for memcpy(3) */
3983+#include <iostream>
3984+#ifndef WIN32
3985+#include <cerrno>
3986+#include <sys/time.h>
3987+#endif /* WIN32 */
3988+
3989+// libcurl
3990+#include <curl/multi.h>
3991+
3992+// Zorba
3993+#include "util/curl_streambuf.h"
3994+
3995+#define ZORBA_LIBCURL_AT_LEAST(MAJOR,MINOR,PATCH) \
3996+ (LIBCURL_VERSION_MAJOR >= (MAJOR) && LIBCURL_VERSION_MINOR >= (MINOR) && LIBCURL_VERSION_PATCH >= (PATCH))
3997+
3998+using namespace std;
3999+
4000+namespace zorba {
4001+namespace curl {
4002+
4003+///////////////////////////////////////////////////////////////////////////////
4004+
4005+exception::exception( char const *function, char const *uri, char const *msg ) :
4006+ std::exception(),
4007+ curl_code_( CURLE_OK ), curlm_code_( CURLM_OK ), msg_( msg )
4008+{
4009+}
4010+
4011+exception::exception( char const *function, char const *uri, CURLcode code ) :
4012+ std::exception(),
4013+ curl_code_( code ), curlm_code_( CURLM_OK ),
4014+ msg_( curl_easy_strerror( code ) )
4015+{
4016+ ostringstream oss;
4017+ oss << " (CURLcode " << (int)code << ')';
4018+ msg_ += oss.str();
4019+}
4020+
4021+exception::exception( char const *function, char const *uri, CURLMcode code ) :
4022+ std::exception(),
4023+ curl_code_( CURLE_OK ), curlm_code_( code ),
4024+ msg_( curl_multi_strerror( code ) )
4025+{
4026+ ostringstream oss;
4027+ oss << " (CURLMcode " << (int)code << ')';
4028+ msg_ += oss.str();
4029+}
4030+
4031+exception::~exception() throw() {
4032+ // out-of-line since it's virtual
4033+}
4034+
4035+const char* exception::what() const throw() {
4036+ return msg_.c_str();
4037+}
4038+
4039+///////////////////////////////////////////////////////////////////////////////
4040+
4041+listener::~listener() {
4042+ // out-of-line since it's virtual
4043+}
4044+
4045+///////////////////////////////////////////////////////////////////////////////
4046+
4047+CURL* create() {
4048+ //
4049+ // Having cURL initialization wrapped by a class and using a singleton static
4050+ // instance guarantees that cURL is initialized exactly once before use and
4051+ // and also is cleaned-up at program termination (when destructors for static
4052+ // objects are called).
4053+ //
4054+ struct curl_initializer {
4055+ curl_initializer() {
4056+ ZORBA_CURL_ASSERT( curl_global_init( CURL_GLOBAL_ALL ) );
4057+ }
4058+ ~curl_initializer() {
4059+ curl_global_cleanup();
4060+ }
4061+ };
4062+ static curl_initializer initializer;
4063+
4064+ if ( CURL *const curl = curl_easy_init() )
4065+ return curl;
4066+ throw exception( "curl_easy_init()", "", "" );
4067+}
4068+
4069+void destroy( CURL *instance ) {
4070+ if ( instance ) {
4071+ curl_easy_reset( instance );
4072+ curl_easy_cleanup( instance );
4073+ }
4074+}
4075+
4076+///////////////////////////////////////////////////////////////////////////////
4077+
4078+streambuf::streambuf() {
4079+ init();
4080+}
4081+
4082+streambuf::streambuf( char const *uri ) {
4083+ init();
4084+ open( uri );
4085+}
4086+
4087+streambuf::streambuf( CURL *curl ) {
4088+ init();
4089+ curl_ = curl;
4090+ curl_init();
4091+ curlm_init();
4092+}
4093+
4094+streambuf::~streambuf() {
4095+ close();
4096+#ifdef WIN32
4097+ closesocket( dummy_socket_ );
4098+#endif /* WIN32 */
4099+ if ( listener_owner_ )
4100+ delete listener_;
4101+}
4102+
4103+void streambuf::close() {
4104+ if ( curl_ ) {
4105+ if ( curlm_ ) {
4106+ curl_multi_remove_handle( curlm_, curl_ );
4107+ curl_multi_cleanup( curlm_ );
4108+ curlm_ = 0;
4109+ }
4110+ curl_destroy();
4111+ }
4112+}
4113+
4114+void streambuf::curl_verbose( bool verbose ) {
4115+ verbose_ = verbose;
4116+ if ( curl_ )
4117+ ZORBA_CURL_ASSERT( curl_easy_setopt( curl_, CURLOPT_VERBOSE, verbose_ ? 1 : 0 ) );
4118+}
4119+
4120+void streambuf::curl_init() {
4121+#if ZORBA_LIBCURL_AT_LEAST(7,25,0)
4122+ ZORBA_CURL_ASSERT( curl_easy_setopt( curl_, CURLOPT_TCP_KEEPALIVE, 1L ) );
4123+#endif
4124+ ZORBA_CURL_ASSERT( curl_easy_setopt( curl_, CURLOPT_WRITEDATA, this ) );
4125+ ZORBA_CURL_ASSERT( curl_easy_setopt( curl_, CURLOPT_WRITEFUNCTION, curl_write_callback ) );
4126+
4127+ if ( verbose_ )
4128+ ZORBA_CURL_ASSERT( curl_easy_setopt( curl_, CURLOPT_VERBOSE, 1 ) );
4129+}
4130+
4131+void streambuf::curl_io( size_t *len_ptr ) {
4132+ *len_ptr = 0;
4133+ while ( curl_running_ && !*len_ptr ) {
4134+ fd_set fd_read, fd_write, fd_except;
4135+ FD_ZERO( &fd_read );
4136+ FD_ZERO( &fd_write );
4137+ FD_ZERO( &fd_except );
4138+ int max_fd = -1;
4139+#ifdef WIN32
4140+ //
4141+ // Windows does not like a call to select where all arguments are 0, so we
4142+ // just add a dummy socket to make the call to select happy.
4143+ //
4144+ FD_SET( dummy_socket_, &fd_read );
4145+#endif /* WIN32 */
4146+ ZORBA_CURLM_ASSERT(
4147+ curl_multi_fdset( curlm_, &fd_read, &fd_write, &fd_except, &max_fd )
4148+ );
4149+
4150+ //
4151+ // Note that the fopen.c sample code is unnecessary at best or wrong at
4152+ // worst; see: http://curl.haxx.se/mail/lib-2011-05/0011.html
4153+ //
4154+ timeval timeout;
4155+ long curl_timeout_ms;
4156+ ZORBA_CURLM_ASSERT( curl_multi_timeout( curlm_, &curl_timeout_ms ) );
4157+ if ( curl_timeout_ms > 0 ) {
4158+ timeout.tv_sec = curl_timeout_ms / 1000;
4159+ timeout.tv_usec = curl_timeout_ms % 1000 * 1000;
4160+ } else {
4161+ //
4162+ // From curl_multi_timeout(3):
4163+ //
4164+ // Note: if libcurl returns a -1 timeout here, it just means that
4165+ // libcurl currently has no stored timeout value. You must not wait
4166+ // too long (more than a few seconds perhaps) before you call
4167+ // curl_multi_perform() again.
4168+ //
4169+ // So we just pick some not-too-long default.
4170+ //
4171+ timeout.tv_sec = 1;
4172+ timeout.tv_usec = 0;
4173+ }
4174+
4175+ switch ( select( max_fd + 1, &fd_read, &fd_write, &fd_except, &timeout ) ) {
4176+ case -1: // select error
4177+#ifdef WIN32
4178+ char err_buf[8];
4179+ sprintf( err_buf, "%d", WSAGetLastError() );
4180+ throw exception( "select()", "", err_buf );
4181+#else
4182+ throw exception( "select()", "", strerror( errno ) );
4183+#endif /* WIN32 */
4184+ case 0: // timeout
4185+ // no break;
4186+ default:
4187+ CURLMcode code;
4188+ do {
4189+ code = curl_multi_perform( curlm_, &curl_running_ );
4190+ } while ( code == CURLM_CALL_MULTI_PERFORM );
4191+ ZORBA_CURLM_ASSERT( code );
4192+ } // switch
4193+ } // while
4194+}
4195+
4196+CURLcode streambuf::curl_multi_info_read( bool throw_on_error ) {
4197+ underflow();
4198+ CURLMsg *msg;
4199+ int msgs_in_q;
4200+ CURLcode code = CURLE_OK;
4201+ while ( (msg = ::curl_multi_info_read( curlm_, &msgs_in_q )) )
4202+ if ( msg->msg == CURLMSG_DONE )
4203+ code = msg->data.result;
4204+ if ( code && throw_on_error )
4205+ throw exception( "curl_multi_info_read()", "", code );
4206+ return code;
4207+}
4208+
4209+size_t streambuf::curl_write_callback( void *ptr, size_t size, size_t nmemb,
4210+ void *data ) {
4211+ size *= nmemb;
4212+ streambuf *const that = static_cast<streambuf*>( data );
4213+
4214+ if ( that->listener_ )
4215+ that->listener_->curl_read( ptr, size );
4216+
4217+ size_t const gbuf_free = that->gbuf_.capacity_ - that->gbuf_.len_;
4218+ if ( size > gbuf_free ) {
4219+ size_t new_capacity = that->gbuf_.capacity_ + size - gbuf_free;
4220+ if ( void *const new_buf = realloc( that->gbuf_.ptr_, new_capacity ) ) {
4221+ that->gbuf_.ptr_ = static_cast<char*>( new_buf );
4222+ that->gbuf_.capacity_ = new_capacity;
4223+ } else
4224+ throw exception( "realloc()", "" );
4225+ }
4226+ ::memcpy( that->gbuf_.ptr_ + that->gbuf_.len_, ptr, size );
4227+ that->gbuf_.len_ += size;
4228+ return size;
4229+}
4230+
4231+void streambuf::init() {
4232+ curl_ = 0;
4233+ curlm_ = 0;
4234+ curl_running_ = 0;
4235+#ifdef WIN32
4236+ dummy_socket_ = socket( AF_INET, SOCK_DGRAM, 0 );
4237+ if ( dummy_socket_ == CURL_SOCKET_BAD || dummy_socket_ == INVALID_SOCKET )
4238+ throw exception( "socket()", "" );
4239+#endif /* WIN32 */
4240+ listener_ = 0;
4241+ listener_owner_ = false;
4242+ verbose_ = false;
4243+}
4244+
4245+void streambuf::curlm_init() {
4246+ //
4247+ // Lie about cURL running initially so the while-loop in curl_io() will run
4248+ // at least once.
4249+ //
4250+ curl_running_ = 1;
4251+
4252+ //
4253+ // Set the "get" pointer to the end (gptr() == egptr()) so a call to
4254+ // underflow() and initial data read will be triggered.
4255+ //
4256+ gbuf_.len_ = gbuf_.capacity_;
4257+ setg( gbuf_.ptr_, gbuf_.ptr_ + gbuf_.len_, gbuf_.ptr_ + gbuf_.len_ );
4258+
4259+ //
4260+ // Clean-up has to be done here with try/catch (as opposed to relying on the
4261+ // destructor) because open() can be called from the constructor. If an
4262+ // exception is thrown, the constructor will not have completed, hence the
4263+ // object will not have been fully constructed; therefore the destructor will
4264+ // not be called.
4265+ //
4266+ try {
4267+ if ( !(curlm_ = curl_multi_init()) )
4268+ throw exception( "curl_multi_init()", "" );
4269+ try {
4270+ ZORBA_CURLM_ASSERT( curl_multi_add_handle( curlm_, curl_ ) );
4271+ }
4272+ catch ( ... ) {
4273+ curl_multi_cleanup( curlm_ );
4274+ curlm_ = 0;
4275+ throw;
4276+ }
4277+ }
4278+ catch ( ... ) {
4279+ curl_destroy();
4280+ throw;
4281+ }
4282+}
4283+
4284+void streambuf::open( char const *uri ) {
4285+ if ( !curl_ ) {
4286+ curl_ = create();
4287+ try {
4288+ curl_init();
4289+ }
4290+ catch ( ... ) {
4291+ curl_destroy();
4292+ throw;
4293+ }
4294+ curlm_init();
4295+ }
4296+ ZORBA_CURL_ASSERT( curl_easy_setopt( curl_, CURLOPT_URL, uri ) );
4297+}
4298+
4299+void streambuf::set_listener( listener *new_listener, bool take_ownership ) {
4300+ if ( listener_owner_ && new_listener != listener_ )
4301+ delete listener_;
4302+ listener_ = new_listener;
4303+ listener_owner_ = take_ownership;
4304+}
4305+
4306+streamsize streambuf::showmanyc() {
4307+ return egptr() - gptr();
4308+}
4309+
4310+streambuf::int_type streambuf::underflow() {
4311+ while ( true ) {
4312+ if ( gptr() < egptr() )
4313+ return traits_type::to_int_type( *gptr() );
4314+ curl_io( &gbuf_.len_ );
4315+ if ( !gbuf_.len_ )
4316+ return traits_type::eof();
4317+ setg( gbuf_.ptr_, gbuf_.ptr_, gbuf_.ptr_ + gbuf_.len_ );
4318+ }
4319+}
4320+
4321+streamsize streambuf::xsgetn( char_type *to, streamsize size ) {
4322+ streamsize return_size = 0;
4323+
4324+ if ( streamsize const gsize = egptr() - gptr() ) {
4325+ streamsize const n = min( gsize, size );
4326+ traits_type::copy( to, gptr(), static_cast<size_t>( n ) );
4327+ gbump( static_cast<int>( n ) );
4328+ to += n;
4329+ size -= n, return_size += n;
4330+ }
4331+
4332+ while ( size > 0 ) {
4333+ streamsize const get = min( (streamsize)gbuf_.capacity_, size );
4334+ curl_io( &gbuf_.len_ );
4335+ if ( !gbuf_.len_ )
4336+ break;
4337+ setg( gbuf_.ptr_, gbuf_.ptr_, gbuf_.ptr_ + gbuf_.len_ );
4338+ streamsize const n = min( (streamsize)gbuf_.len_, size );
4339+ traits_type::copy( to, gptr(), n );
4340+ gbump( static_cast<int>( n ) );
4341+ to += n;
4342+ size -= n, return_size += n;
4343+ }
4344+ return return_size;
4345+}
4346+
4347+///////////////////////////////////////////////////////////////////////////////
4348+
4349+} // namespace curl
4350+} // namespace zorba
4351+/* vim:set et sw=2 ts=2: */
4352
4353=== added file 'modules/util-curl/src/util-curl.cpp'
4354--- modules/util-curl/src/util-curl.cpp 1970-01-01 00:00:00 +0000
4355+++ modules/util-curl/src/util-curl.cpp 2014-01-13 19:25:59 +0000
4356@@ -0,0 +1,59 @@
4357+/*
4358+ * Copyright 2006-2008 The FLWOR Foundation.
4359+ *
4360+ * Licensed under the Apache License, Version 2.0 (the "License");
4361+ * you may not use this file except in compliance with the License.
4362+ * You may obtain a copy of the License at
4363+ *
4364+ * http://www.apache.org/licenses/LICENSE-2.0
4365+ *
4366+ * Unless required by applicable law or agreed to in writing, software
4367+ * distributed under the License is distributed on an "AS IS" BASIS,
4368+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
4369+ * See the License for the specific language governing permissions and
4370+ * limitations under the License.
4371+ */
4372+
4373+#include <zorba/external_module.h>
4374+
4375+namespace zorba {
4376+namespace util_curl {
4377+
4378+///////////////////////////////////////////////////////////////////////////////
4379+
4380+class module : public ExternalModule {
4381+public:
4382+ // inherited
4383+ virtual void destroy();
4384+ virtual ExternalFunction* getExternalFunction( String const& );
4385+ virtual String getURI() const;
4386+};
4387+
4388+void module::destroy() {
4389+ delete this;
4390+}
4391+
4392+ExternalFunction* module::getExternalFunction( String const& ) {
4393+ return 0;
4394+}
4395+
4396+String module::getURI() const {
4397+ return "http://zorba.io/modules/util-curl";
4398+}
4399+
4400+///////////////////////////////////////////////////////////////////////////////
4401+
4402+} // namespace util_curl
4403+} // namespace zorba
4404+
4405+#ifdef WIN32
4406+# define DLL_EXPORT __declspec(dllexport)
4407+#else
4408+# define DLL_EXPORT __attribute__ ((visibility("default")))
4409+#endif
4410+
4411+extern "C" DLL_EXPORT zorba::ExternalModule* createModule() {
4412+ return new zorba::util_curl::module();
4413+}
4414+
4415+/* vim:set et sw=2 ts=2: */
4416
4417=== added file 'modules/util-curl/src/util-curl.xq'
4418--- modules/util-curl/src/util-curl.xq 1970-01-01 00:00:00 +0000
4419+++ modules/util-curl/src/util-curl.xq 2014-01-13 19:25:59 +0000
4420@@ -0,0 +1,34 @@
4421+xquery version "1.0";
4422+
4423+(:
4424+ : Copyright 2006-2013 The FLWOR Foundation.
4425+ :
4426+ : Licensed under the Apache License, Version 2.0 (the "License");
4427+ : you may not use this file except in compliance with the License.
4428+ : You may obtain a copy of the License at
4429+ :
4430+ : http://www.apache.org/licenses/LICENSE-2.0
4431+ :
4432+ : Unless required by applicable law or agreed to in writing, software
4433+ : distributed under the License is distributed on an "AS IS" BASIS,
4434+ : WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
4435+ : See the License for the specific language governing permissions and
4436+ : limitations under the License.
4437+ :)
4438+
4439+(:~
4440+ : This module provides common functionality for modules that use libcurl.
4441+ : <p/>
4442+ : Modules using libcurl implementations must import this module
4443+ : to specify the dependency.
4444+ :
4445+ : @author Paul J. Lucas
4446+ : @library <a href="http://curl.haxx.se/download.html">libcurl</a>
4447+ : @project Zorba/cURL Utility
4448+ :)
4449+module namespace util-curl = "http://zorba.io/modules/util-curl";
4450+
4451+declare namespace ver = "http://zorba.io/options/versioning";
4452+declare option ver:module-version "1.0";
4453+
4454+(: vim:set et sw=2 ts=2: :)
4455
4456=== modified file 'src/api/CMakeLists.txt'
4457--- src/api/CMakeLists.txt 2013-09-16 09:08:27 +0000
4458+++ src/api/CMakeLists.txt 2014-01-13 19:25:59 +0000
4459@@ -59,6 +59,7 @@
4460 auditimpl.cpp
4461 store_consts.cpp
4462 streambuf.cpp
4463+ mem_streambuf.cpp
4464 transcode_streambuf.cpp
4465 uuid.cpp
4466 module_info_impl.cpp
4467
4468=== modified file 'src/api/base64_util.cpp'
4469--- src/api/base64_util.cpp 2013-07-30 18:35:31 +0000
4470+++ src/api/base64_util.cpp 2014-01-13 19:25:59 +0000
4471@@ -22,8 +22,8 @@
4472
4473 // Zorba
4474 #include <zorba/util/base64_util.h>
4475+#include <zorba/util/mem_streambuf.h>
4476 #include "util/ascii_util.h"
4477-#include "util/mem_streambuf.h"
4478 #include "util/string_util.h"
4479
4480 using namespace std;
4481
4482=== modified file 'src/api/hexbinary_util.cpp'
4483--- src/api/hexbinary_util.cpp 2013-08-02 14:55:29 +0000
4484+++ src/api/hexbinary_util.cpp 2014-01-13 19:25:59 +0000
4485@@ -22,7 +22,7 @@
4486
4487 // Zorba
4488 #include <zorba/util/hexbinary_util.h>
4489-#include "util/mem_streambuf.h"
4490+#include <zorba/util/mem_streambuf.h>
4491 #include "util/string_util.h"
4492
4493 using namespace std;
4494
4495=== renamed file 'src/util/mem_streambuf.cpp' => 'src/api/mem_streambuf.cpp'
4496--- src/util/mem_streambuf.cpp 2013-08-01 00:31:20 +0000
4497+++ src/api/mem_streambuf.cpp 2014-01-13 19:25:59 +0000
4498@@ -18,11 +18,10 @@
4499 #include <cstring> /* for memcpy(3) */
4500
4501 #include <zorba/internal/cxx_util.h>
4502+#include <zorba/util/mem_streambuf.h>
4503
4504 #include "diagnostics/assert.h"
4505
4506-#include "mem_streambuf.h"
4507-
4508 using namespace std;
4509
4510 namespace zorba {
4511
4512=== modified file 'src/runtime/csv/pregenerated/csv.h'
4513--- src/runtime/csv/pregenerated/csv.h 2013-08-27 23:56:50 +0000
4514+++ src/runtime/csv/pregenerated/csv.h 2014-01-13 19:25:59 +0000
4515@@ -31,9 +31,9 @@
4516 #include "runtime/base/narybase.h"
4517 #include <sstream>
4518 #include <vector>
4519+#include <zorba/util/mem_streambuf.h>
4520 #include "runtime/csv/csv_util.h"
4521 #include "util/csv_parser.h"
4522-#include "util/mem_streambuf.h"
4523 #include "zorbatypes/zstring.h"
4524
4525
4526
4527=== modified file 'src/runtime/json/json_impl.cpp'
4528--- src/runtime/json/json_impl.cpp 2013-12-12 18:41:52 +0000
4529+++ src/runtime/json/json_impl.cpp 2014-01-13 19:25:59 +0000
4530@@ -20,12 +20,12 @@
4531
4532 #include <zorba/diagnostic_list.h>
4533 #include <zorba/internal/cxx_util.h>
4534+#include <zorba/util/mem_streambuf.h>
4535
4536 #include "runtime/json/json.h"
4537 #include "store/api/item_factory.h"
4538 #include "system/globalenv.h"
4539 #include "util/ascii_util.h"
4540-#include "util/mem_streambuf.h"
4541 #include "util/stream_util.h"
4542
4543 #include "jsonml_array.h"
4544
4545=== modified file 'src/runtime/json/snelson.cpp'
4546--- src/runtime/json/snelson.cpp 2013-08-25 14:32:02 +0000
4547+++ src/runtime/json/snelson.cpp 2014-01-13 19:25:59 +0000
4548@@ -21,6 +21,7 @@
4549 #include <zorba/diagnostic_list.h>
4550 #include <zorba/internal/cxx_util.h>
4551 #include <zorba/store_consts.h>
4552+#include <zorba/util/mem_streambuf.h>
4553
4554 #include "runtime/json/json.h"
4555 #include "store/api/item_factory.h"
4556@@ -28,7 +29,6 @@
4557 #include "types/root_typemanager.h"
4558 #include "types/typeops.h"
4559 #include "util/json_parser.h"
4560-#include "util/mem_streambuf.h"
4561 #include "util/stl_util.h"
4562 #include "zorbatypes/decimal.h"
4563 #include "zorbatypes/float.h"
4564
4565=== modified file 'src/runtime/spec/csv/csv.xml'
4566--- src/runtime/spec/csv/csv.xml 2013-08-27 23:56:50 +0000
4567+++ src/runtime/spec/csv/csv.xml 2014-01-13 19:25:59 +0000
4568@@ -8,9 +8,9 @@
4569 <zorba:header>
4570 <zorba:include form="Angle-bracket">sstream</zorba:include>
4571 <zorba:include form="Angle-bracket">vector</zorba:include>
4572+ <zorba:include form="Angle-bracket">zorba/util/mem_streambuf.h</zorba:include>
4573 <zorba:include form="Quoted">runtime/csv/csv_util.h</zorba:include>
4574 <zorba:include form="Quoted">util/csv_parser.h</zorba:include>
4575- <zorba:include form="Quoted">util/mem_streambuf.h</zorba:include>
4576 <zorba:include form="Quoted">zorbatypes/zstring.h</zorba:include>
4577 </zorba:header>
4578
4579
4580=== modified file 'src/util/CMakeLists.txt'
4581--- src/util/CMakeLists.txt 2013-08-21 23:52:57 +0000
4582+++ src/util/CMakeLists.txt 2014-01-13 19:25:59 +0000
4583@@ -22,7 +22,6 @@
4584 json_parser.cpp
4585 json_util.cpp
4586 locale.cpp
4587- mem_streambuf.cpp
4588 stream_util.cpp
4589 string_util.cpp
4590 time_util.cpp

Subscribers

People subscribed via source and target branches