Merge lp:~barry-leslie/drizzle/drizzle_pbmsV2 into lp:~drizzle-trunk/drizzle/development

Proposed by Barry Leslie on 2011-04-06
Status: Work in progress
Proposed branch: lp:~barry-leslie/drizzle/drizzle_pbmsV2
Merge into: lp:~drizzle-trunk/drizzle/development
Diff against target: 42221 lines (+16542/-20847)
128 files modified
plugin/pbms/plugin.am (+0/-12)
plugin/pbms/plugin.ini (+121/-112)
plugin/pbms/src/TransTest.cc (+0/-953)
plugin/pbms/src/alias_ms.cc (+0/-1092)
plugin/pbms/src/alias_ms.h (+0/-350)
plugin/pbms/src/api_ms.cc (+0/-270)
plugin/pbms/src/backup_ms.cc (+0/-733)
plugin/pbms/src/backup_ms.h (+0/-186)
plugin/pbms/src/cloud/cloud_ms.cc (+162/-271)
plugin/pbms/src/cloud/cloud_ms.h (+61/-139)
plugin/pbms/src/compactor_ms.cc (+0/-343)
plugin/pbms/src/compactor_ms.h (+0/-64)
plugin/pbms/src/cslib/CSConfig.h (+10/-10)
plugin/pbms/src/cslib/CSDefs.h (+2/-1)
plugin/pbms/src/cslib/CSEncode.h (+1/-1)
plugin/pbms/src/cslib/CSException.h (+1/-0)
plugin/pbms/src/cslib/CSFile.cc (+128/-0)
plugin/pbms/src/cslib/CSFile.h (+82/-1)
plugin/pbms/src/cslib/CSGlobal.h (+1/-0)
plugin/pbms/src/cslib/CSHTTPStream.h (+1/-0)
plugin/pbms/src/cslib/CSIdSpace.cc (+366/-0)
plugin/pbms/src/cslib/CSIdSpace.h (+89/-0)
plugin/pbms/src/cslib/CSLog.cc (+2/-5)
plugin/pbms/src/cslib/CSMemory.cc (+3/-0)
plugin/pbms/src/cslib/CSMutex.cc (+1/-2)
plugin/pbms/src/cslib/CSObject.h (+3/-1)
plugin/pbms/src/cslib/CSPath.cc (+24/-5)
plugin/pbms/src/cslib/CSPath.h (+2/-0)
plugin/pbms/src/cslib/CSS3Protocol.cc (+22/-11)
plugin/pbms/src/cslib/CSSocket.cc (+44/-4)
plugin/pbms/src/cslib/CSSocket.h (+12/-2)
plugin/pbms/src/cslib/CSStorage.cc (+30/-6)
plugin/pbms/src/cslib/CSStorage.h (+8/-3)
plugin/pbms/src/cslib/CSStrUtil.cc (+0/-1)
plugin/pbms/src/cslib/CSStrUtil.h (+0/-2)
plugin/pbms/src/cslib/CSStream.cc (+66/-0)
plugin/pbms/src/cslib/CSStream.h (+37/-3)
plugin/pbms/src/cslib/CSString.cc (+1/-3)
plugin/pbms/src/cslib/CSString.h (+3/-0)
plugin/pbms/src/cslib/CSSys.h (+2/-0)
plugin/pbms/src/cslib/CSSys_unix.cc (+11/-0)
plugin/pbms/src/cslib/CSSys_win.cc (+7/-0)
plugin/pbms/src/cslib/CSThread.cc (+5/-10)
plugin/pbms/src/cslib/CSThread.h (+6/-0)
plugin/pbms/src/daemon/pbmsdaemon_ms.cc (+1084/-19)
plugin/pbms/src/daemon/pbmsdaemon_ms.h (+133/-11)
plugin/pbms/src/database/database_ms.cc (+630/-1532)
plugin/pbms/src/database/database_ms.h (+101/-182)
plugin/pbms/src/discover_ms.cc (+0/-1390)
plugin/pbms/src/discover_ms.h (+0/-96)
plugin/pbms/src/includes/defs_ms.h (+40/-14)
plugin/pbms/src/includes/pbms.h (+32/-708)
plugin/pbms/src/includes/pbms_config.h (+57/-0)
plugin/pbms/src/includes/version_ms.h (+1/-1)
plugin/pbms/src/lib/pbmslib.cc (+1844/-0)
plugin/pbms/src/lib/pbmslib.h (+99/-85)
plugin/pbms/src/mysql_ms.cc (+0/-104)
plugin/pbms/src/mysql_ms.h (+0/-40)
plugin/pbms/src/network/connection_handler_ms.cc (+289/-168)
plugin/pbms/src/network/connection_handler_ms.h (+9/-3)
plugin/pbms/src/network/network_ms.cc (+7/-8)
plugin/pbms/src/network/network_ms.h (+1/-1)
plugin/pbms/src/open_table_ms.cc (+0/-1121)
plugin/pbms/src/open_table_ms.h (+0/-213)
plugin/pbms/src/pbms_enabled.cc (+0/-421)
plugin/pbms/src/pbms_enabled.h (+0/-133)
plugin/pbms/src/plugin/drizzle/events_ms.cc (+15/-14)
plugin/pbms/src/plugin/drizzle/events_ms.h (+7/-2)
plugin/pbms/src/plugin/engine_ms.cc (+291/-699)
plugin/pbms/src/plugin/engine_ms.h (+8/-47)
plugin/pbms/src/plugin/ha_pbms.cc (+242/-373)
plugin/pbms/src/plugin/ha_pbms.h (+18/-14)
plugin/pbms/src/plugin/parameters_ms.cc (+90/-101)
plugin/pbms/src/plugin/parameters_ms.h (+5/-6)
plugin/pbms/src/plugin/plugin_ms.cc (+47/-19)
plugin/pbms/src/repository/data/blob_repository_ms.cc (+1203/-0)
plugin/pbms/src/repository/data/blob_repository_ms.h (+162/-0)
plugin/pbms/src/repository/data/repository_file_ms.cc (+790/-0)
plugin/pbms/src/repository/data/repository_file_ms.h (+108/-0)
plugin/pbms/src/repository/data/repository_ms.cc (+535/-1606)
plugin/pbms/src/repository/data/repository_ms.h (+136/-231)
plugin/pbms/src/repository/index/repo_index_ms.cc (+862/-0)
plugin/pbms/src/repository/index/repo_index_ms.h (+206/-0)
plugin/pbms/src/repository/logs/temp_log_ms.cc (+399/-331)
plugin/pbms/src/repository/logs/temp_log_ms.h (+49/-81)
plugin/pbms/src/repository/references/blob_refs.cc (+298/-0)
plugin/pbms/src/repository/references/blob_refs.h (+100/-0)
plugin/pbms/src/repository/references/bloburl.cc (+328/-0)
plugin/pbms/src/repository/references/bloburl.h (+217/-0)
plugin/pbms/src/repository/references/metadata_ms.cc (+296/-1)
plugin/pbms/src/repository/references/metadata_ms.h (+60/-83)
plugin/pbms/src/repository/references/table_ms.cc (+564/-339)
plugin/pbms/src/repository/references/table_ms.h (+60/-82)
plugin/pbms/src/systab_backup_ms.cc (+0/-689)
plugin/pbms/src/systab_backup_ms.h (+0/-78)
plugin/pbms/src/systab_dump_ms.cc (+0/-586)
plugin/pbms/src/systab_dump_ms.h (+0/-63)
plugin/pbms/src/systab_enabled_ms.cc (+0/-192)
plugin/pbms/src/systab_enabled_ms.h (+0/-53)
plugin/pbms/src/systab_variable_ms.cc (+0/-791)
plugin/pbms/src/systab_variable_ms.h (+0/-73)
plugin/pbms/src/system_tables/pbms_blob.cc (+134/-0)
plugin/pbms/src/system_tables/pbms_blob.h (+61/-0)
plugin/pbms/src/system_tables/pbms_cloud.cc (+233/-251)
plugin/pbms/src/system_tables/pbms_cloud.h (+12/-12)
plugin/pbms/src/system_tables/pbms_metadata.cc (+470/-0)
plugin/pbms/src/system_tables/pbms_metadata.h (+83/-0)
plugin/pbms/src/system_tables/pbms_metadata_header.cc (+104/-87)
plugin/pbms/src/system_tables/pbms_metadata_header.h (+15/-15)
plugin/pbms/src/system_tables/pbms_reference.cc (+387/-0)
plugin/pbms/src/system_tables/pbms_reference.h (+75/-0)
plugin/pbms/src/system_tables/pbms_repository.cc (+506/-0)
plugin/pbms/src/system_tables/pbms_repository.h (+85/-0)
plugin/pbms/src/system_tables/pbms_server.cc (+468/-0)
plugin/pbms/src/system_tables/pbms_server.h (+115/-0)
plugin/pbms/src/system_tables/systab_util_ms.cc (+98/-12)
plugin/pbms/src/system_tables/systab_util_ms.h (+72/-6)
plugin/pbms/src/system_tables/system_table_ms.cc (+270/-2136)
plugin/pbms/src/system_tables/system_table_ms.h (+124/-251)
plugin/pbms/src/transactions/trans_cache_ms.cc (+1/-3)
plugin/pbms/src/transactions/trans_log_ms.cc (+9/-14)
plugin/pbms/src/transactions/trans_log_ms.h (+2/-18)
plugin/pbms/src/transactions/transaction_ms.cc (+125/-26)
plugin/pbms/src/transactions/transaction_ms.h (+4/-9)
plugin/pbms/src/udf_ms.cc (+0/-549)
plugin/pbms/tests/r/basic.result (+163/-19)
plugin/pbms/tests/t/basic.test (+187/-37)
po/POTFILES.in (+2/-2)
To merge this branch: bzr merge lp:~barry-leslie/drizzle/drizzle_pbmsV2
Reviewer Review Type Date Requested Status
Monty Taylor 2011-04-06 Needs Fixing on 2011-04-07
Review via email: mp+56493@code.launchpad.net

Description of the Change

This is the new version of PBMS which is pretty much a rewrite of the old version. This version fixes some limitations in the old version and is designed to work with replication.

It is NOT backward compatible with the old version. If this is a problem for anyone they can contact me and I will help them porting from the old version to the new one.

To post a comment you must log in.
Monty Taylor (mordred) wrote :
review: Needs Fixing
Barry Leslie (barry-leslie) wrote :

This is a test case problem, it is assuming the tests are being run in a new
database.

I will see if I can change this so that it will give the expected results
when run twice.

On 4/7/11 9:47 AM, "Monty Taylor" <email address hidden> wrote:

> Review: Needs Fixing
> Hey Barry,
>
> This branch fails the "run tests twice" job:
>
> http://jenkins.drizzle.org/view/Drizzle-build/job/drizzle-build-repeat-tests-t
> wice/387/console
>

-------------------------------------------------------------------------
Barry Leslie

Barry Leslie (barry-leslie) wrote :

"run tests twice" now works.

Barry Leslie (barry-leslie) wrote :

I am continuing to work on this branch and will let people know when I think it is ready to be merged.

1979. By Barry Leslie on 2011-06-01

Merged with drizzle trunk

1980. By Barry Leslie on 2011-06-01

Use correct charset.h

1981. By Barry Leslie on 2011-06-01

Merged changes from the main pbms project.

Unmerged revisions

1981. By Barry Leslie on 2011-06-01

Merged changes from the main pbms project.

1980. By Barry Leslie on 2011-06-01

Use correct charset.h

1979. By Barry Leslie on 2011-06-01

Merged with drizzle trunk

1978. By Barry Leslie on 2011-04-07

Updated test case so it can be run multiple times.

1977. By Barry Leslie on 2011-04-05

Fixed fo renamed pbms files in POTFILES.in

1976. By Barry Leslie on 2011-04-04

fixed un initialized variable warning.

1975. By Barry Leslie on 2011-04-04

Code cleanup

1974. By Barry Leslie on 2011-04-04

Merged in lp:drizzle.

1973. By Barry Leslie on 2011-04-04

Added warning message to expected test out put until bug 746720 is fixed.

1972. By Barry Leslie on 2011-03-31

Removed an invalid test from the test cases.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== removed file 'plugin/pbms/plugin.am'
2--- plugin/pbms/plugin.am 2010-07-14 22:58:28 +0000
3+++ plugin/pbms/plugin.am 1970-01-01 00:00:00 +0000
4@@ -1,12 +0,0 @@
5-
6-EXTRA_DIST+= \
7- plugin/pbms/src/TransTest.cc \
8- plugin/pbms/src/alias_ms.cc \
9- plugin/pbms/src/alias_ms.h \
10- plugin/pbms/src/discover_ms.cc \
11- plugin/pbms/src/pbms_enabled.cc \
12- plugin/pbms/src/pbms_enabled.h \
13- plugin/pbms/src/systab_enabled_ms.cc \
14- plugin/pbms/src/systab_enabled_ms.h \
15- plugin/pbms/src/udf_ms.cc
16-
17
18=== modified file 'plugin/pbms/plugin.ini'
19--- plugin/pbms/plugin.ini 2010-12-18 04:43:40 +0000
20+++ plugin/pbms/plugin.ini 2011-06-01 23:57:43 +0000
21@@ -19,121 +19,132 @@
22 title=PrimeBase Media Stream Daemon
23 description=Provides BLOB streaming service for storage engines
24 load_by_default=no
25-cxxflags=-DDRIZZLED -DPBMS_VERSION=0.5.14-beta
26+cxxflags= -DDRIZZLED -DPBMS_VERSION=0.6.01-beta -I$(top_srcdir)/plugin/pbms/src
27 build_conditional="${ac_cv_libcurl}" = "yes"
28 ldflags= ${LTLIBCURL}
29
30 # disabled=yes
31
32-sources=src/plugin_ms.cc
33- src/backup_ms.cc
34- src/cloud_ms.cc
35- src/compactor_ms.cc
36- src/connection_handler_ms.cc
37- src/database_ms.cc
38- src/engine_ms.cc
39- src/events_ms.cc
40- src/mysql_ms.cc
41- src/network_ms.cc
42- src/open_table_ms.cc
43- src/parameters_ms.cc
44- src/pbmsdaemon_ms.cc
45- src/ha_pbms.cc
46- src/repository_ms.cc
47- src/systab_backup_ms.cc
48- src/systab_cloud_ms.cc
49- src/systab_dump_ms.cc
50- src/systab_httpheader_ms.cc
51- src/systab_util_ms.cc
52- src/systab_variable_ms.cc
53- src/system_table_ms.cc
54- src/table_ms.cc
55- src/temp_log_ms.cc
56- src/transaction_ms.cc
57- src/trans_cache_ms.cc
58- src/trans_log_ms.cc
59- src/cslib/CSDirectory.cc
60- src/cslib/CSEncode.cc
61- src/cslib/CSException.cc
62- src/cslib/CSFile.cc
63- src/cslib/CSHTTPStream.cc
64- src/cslib/CSLog.cc
65- src/cslib/CSMd5.cc
66- src/cslib/CSMemory.cc
67- src/cslib/CSMutex.cc
68- src/cslib/CSObject.cc
69- src/cslib/CSPath.cc
70- src/cslib/CSS3Protocol.cc
71- src/cslib/CSSha1.cc
72- src/cslib/CSSocket.cc
73- src/cslib/CSStorage.cc
74- src/cslib/CSStream.cc
75- src/cslib/CSString.cc
76- src/cslib/CSStrUtil.cc
77- src/cslib/CSThread.cc
78- src/cslib/CSTime.cc
79- src/cslib/CSUTF8.cc
80- src/cslib/CSXML.cc
81- src/cslib/CSSys_unix.cc
82-
83-headers=src/backup_ms.h
84- src/cloud_ms.h
85- src/compactor_ms.h
86- src/connection_handler_ms.h
87- src/cslib/CSConfig.h
88- src/cslib/CSDefs.h
89- src/cslib/CSDirectory.h
90- src/cslib/CSEncode.h
91- src/cslib/CSException.h
92- src/cslib/CSFile.h
93- src/cslib/CSGlobal.h
94- src/cslib/CSHTTPStream.h
95- src/cslib/CSLog.h
96- src/cslib/CSMd5.h
97- src/cslib/CSMemory.h
98- src/cslib/CSMutex.h
99- src/cslib/CSObject.h
100- src/cslib/CSPath.h
101- src/cslib/CSS3Protocol.h
102- src/cslib/CSSha1.h
103- src/cslib/CSSocket.h
104- src/cslib/CSStorage.h
105- src/cslib/CSStream.h
106- src/cslib/CSString.h
107- src/cslib/CSStrUtil.h
108- src/cslib/CSThread.h
109- src/cslib/CSTime.h
110- src/cslib/CSUTF8.h
111- src/cslib/CSXML.h
112- src/cslib/CSSys.h
113- src/database_ms.h
114- src/defs_ms.h
115- src/discover_ms.h
116- src/engine_ms.h
117- src/events_ms.h
118- src/ha_pbms.h
119- src/metadata_ms.h
120- src/mysql_ms.h
121- src/network_ms.h
122- src/open_table_ms.h
123- src/parameters_ms.h
124- src/pbmsdaemon_ms.h
125- src/pbms.h
126- src/pbmslib.h
127- src/repository_ms.h
128- src/systab_backup_ms.h
129- src/systab_cloud_ms.h
130- src/systab_dump_ms.h
131- src/systab_httpheader_ms.h
132- src/systab_util_ms.h
133- src/systab_variable_ms.h
134- src/system_table_ms.h
135- src/table_ms.h
136- src/temp_log_ms.h
137- src/transaction_ms.h
138- src/trans_cache_ms.h
139- src/trans_log_ms.h
140- src/version_ms.h
141+sources=src/plugin/plugin_ms.cc
142+ src/cloud/cloud_ms.cc
143+ src/daemon/pbmsdaemon_ms.cc
144+ src/database/database_ms.cc
145+ src/network/connection_handler_ms.cc
146+ src/network/network_ms.cc
147+ src/plugin/drizzle/events_ms.cc
148+ src/plugin/engine_ms.cc
149+ src/plugin/ha_pbms.cc
150+ src/plugin/parameters_ms.cc
151+ src/repository/data/blob_repository_ms.cc
152+ src/repository/data/repository_file_ms.cc
153+ src/repository/data/repository_ms.cc
154+ src/repository/index/repo_index_ms.cc
155+ src/repository/logs/temp_log_ms.cc
156+ src/repository/references/blob_refs.cc
157+ src/repository/references/bloburl.cc
158+ src/repository/references/metadata_ms.cc
159+ src/repository/references/table_ms.cc
160+ src/system_tables/pbms_blob.cc
161+ src/system_tables/pbms_cloud.cc
162+ src/system_tables/pbms_metadata.cc
163+ src/system_tables/pbms_metadata_header.cc
164+ src/system_tables/pbms_reference.cc
165+ src/system_tables/pbms_repository.cc
166+ src/system_tables/pbms_server.cc
167+ src/system_tables/systab_util_ms.cc
168+ src/system_tables/system_table_ms.cc
169+ src/transactions/trans_cache_ms.cc
170+ src/transactions/trans_log_ms.cc
171+ src/transactions/transaction_ms.cc
172+ src/cslib/CSDirectory.cc
173+ src/cslib/CSEncode.cc
174+ src/cslib/CSException.cc
175+ src/cslib/CSFile.cc
176+ src/cslib/CSHTTPStream.cc
177+ src/cslib/CSLog.cc
178+ src/cslib/CSMd5.cc
179+ src/cslib/CSMemory.cc
180+ src/cslib/CSMutex.cc
181+ src/cslib/CSObject.cc
182+ src/cslib/CSPath.cc
183+ src/cslib/CSS3Protocol.cc
184+ src/cslib/CSSha1.cc
185+ src/cslib/CSSocket.cc
186+ src/cslib/CSStorage.cc
187+ src/cslib/CSStream.cc
188+ src/cslib/CSString.cc
189+ src/cslib/CSStrUtil.cc
190+ src/cslib/CSThread.cc
191+ src/cslib/CSTime.cc
192+ src/cslib/CSUTF8.cc
193+ src/cslib/CSXML.cc
194+ src/cslib/CSSys_unix.cc
195+ src/cslib/CSIdSpace.cc
196+
197+headers=src/cloud/cloud_ms.h
198+ src/cslib/CSConfig.h
199+ src/cslib/CSDefs.h
200+ src/cslib/CSDirectory.h
201+ src/cslib/CSEncode.h
202+ src/cslib/CSException.h
203+ src/cslib/CSFile.h
204+ src/cslib/CSGlobal.h
205+ src/cslib/CSHTTPStream.h
206+ src/cslib/CSIdSpace.h
207+ src/cslib/CSLog.h
208+ src/cslib/CSMd5.h
209+ src/cslib/CSMemory.h
210+ src/cslib/CSMutex.h
211+ src/cslib/CSObject.h
212+ src/cslib/CSPath.h
213+ src/cslib/CSS3Protocol.h
214+ src/cslib/CSSha1.h
215+ src/cslib/CSSocket.h
216+ src/cslib/CSStorage.h
217+ src/cslib/CSStream.h
218+ src/cslib/CSString.h
219+ src/cslib/CSStrUtil.h
220+ src/cslib/CSSys.h
221+ src/cslib/CSThread.h
222+ src/cslib/CSTime.h
223+ src/cslib/CSUTF8.h
224+ src/cslib/CSXML.h
225+ src/daemon/pbmsdaemon_ms.h
226+ src/database/database_ms.h
227+ src/includes/defs_ms.h
228+ src/includes/pbms_config.h
229+ src/includes/pbms.h
230+ src/includes/version_ms.h
231+ src/lib/pbmslib.h
232+ src/network/connection_handler_ms.h
233+ src/network/network_ms.h
234+ src/plugin/drizzle/events_ms.h
235+ src/plugin/engine_ms.h
236+ src/plugin/ha_pbms.h
237+ src/plugin/parameters_ms.h
238+ src/repository/data/blob_repository_ms.h
239+ src/repository/data/repository_file_ms.h
240+ src/repository/data/repository_ms.h
241+ src/repository/index/repo_index_ms.h
242+ src/repository/logs/temp_log_ms.h
243+ src/repository/references/blob_refs.h
244+ src/repository/references/bloburl.h
245+ src/repository/references/metadata_ms.h
246+ src/repository/references/table_ms.h
247+ src/system_tables/pbms_blob.h
248+ src/system_tables/pbms_cloud.h
249+ src/system_tables/pbms_metadata.h
250+ src/system_tables/pbms_metadata_header.h
251+ src/system_tables/pbms_reference.h
252+ src/system_tables/pbms_repository.h
253+ src/system_tables/pbms_server.h
254+ src/system_tables/systab_util_ms.h
255+ src/system_tables/system_table_ms.h
256+ src/transactions/trans_cache_ms.h
257+ src/transactions/trans_log_ms.h
258+ src/transactions/transaction_ms.h
259+
260+
261
262
263 extra_dist=AUTHORS
264@@ -141,6 +152,4 @@
265 ChangeLog
266 README
267 TODO
268- src/api_ms.cc
269- src/metadata_ms.cc
270 src/cslib/CSSys_win.cc
271
272=== removed file 'plugin/pbms/src/TransTest.cc'
273--- plugin/pbms/src/TransTest.cc 2010-12-18 04:43:40 +0000
274+++ plugin/pbms/src/TransTest.cc 1970-01-01 00:00:00 +0000
275@@ -1,953 +0,0 @@
276-/* Copyright (C) 2009 PrimeBase Technologies GmbH, Germany
277- *
278- * PrimeBase Media Stream for MySQL
279- *
280- * This program is free software; you can redistribute it and/or modify
281- * it under the terms of the GNU General Public License as published by
282- * the Free Software Foundation; either version 2 of the License, or
283- * (at your option) any later version.
284- *
285- * This program is distributed in the hope that it will be useful,
286- * but WITHOUT ANY WARRANTY; without even the implied warranty of
287- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
288- * GNU General Public License for more details.
289- *
290- * You should have received a copy of the GNU General Public License
291- * along with this program; if not, write to the Free Software
292- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
293- *
294- * Barry Leslie
295- *
296- * 2009-06-17
297- *
298- * H&G2JCtL
299- *
300- * PBMS transaction handling test driver.
301- *
302- * This is a test driver for the PBMS transaction log. It uses 2 tables in a database and
303- * inserts transaction records into 1 while writing them to the transaction log. The transaction
304- * log reader thread reads the transactions from the log and writes them to the second table.
305- * After a recovery the 2 tables should be identical.
306- *
307- * Built in crash points can be triggered to test that the recovery works correctly.
308- *
309- */
310-
311-#ifdef UNIT_TEST
312-
313-#include <stdlib.h>
314-#include <stdio.h>
315-#include <unistd.h>
316-#include <string.h>
317-#include <ctype.h>
318-#include <inttypes.h>
319-
320-#include "cslib/CSConfig.h"
321-#include "cslib/CSGlobal.h"
322-#include "cslib/CSThread.h"
323-#include "cslib/CSStrUtil.h"
324-#include "cslib/CSStorage.h"
325-
326-#include "trans_cache_ms.h"
327-#include "trans_log_ms.h"
328-
329-#include "mysql.h"
330-
331-#define CREATE_TABLE_BODY "\
332- (\
333- blob_ref INT NOT NULL AUTO_INCREMENT,\
334- tab_id INT NOT NULL,\
335- blob_id BIGINT NOT NULL, \
336- committed BOOLEAN NOT NULL DEFAULT 0, \
337- PRIMARY KEY (blob_ref, tab_id)\
338-)\
339-ENGINE = INNODB\
340-"
341-#ifdef LOG_TABLE
342-#undef LOG_TABLE
343-#endif
344-
345-#define LOG_TABLE "translog"
346-#define REF_TABLE "transref_%d"
347-#define MAX_THREADS 20
348-
349-#define A_DB_ID 123
350-
351-#define TEST_DATABASE_NAME "TransTest"
352-static const char *user_name = "root";
353-static const char *user_passwd = "";
354-static int port = 3306;
355-static const char *host = "localhost";
356-static int nap_time = 1000;
357-static int max_transaction = 10; // The maximum number of records generated per transaction
358-static bool dump_log = false, overflow_crash = false;
359-static int crash_site = 0; // The location to crash at.
360-static int num_threads = 1; // The number of writer threads.
361-//static int rate = 1000; // The maximum transactions per second to allow.
362-static time_t timeout = 60; // How long to run for before crashing or shutting down.
363-static bool revover_only = false;
364-static bool recreate = false;
365-
366-static uint32_t cache_size = 0, log_size = 0;
367-
368-static MSTrans *trans_log;
369-
370-static CSThreadList *thread_list;
371-
372-static MYSQL *new_connection(bool check_for_db);
373-
374-static CSThread *main_thread;
375-
376-//------------------------------------------------
377-class TransTestThread : public CSDaemon {
378-public:
379- TransTestThread():
380- CSDaemon(thread_list),
381- count(0),
382- myActivity(0),
383- log(NULL),
384- stopit(false),
385- finished(false),
386- mysql(NULL)
387- {}
388-
389- ~TransTestThread()
390- {
391- if (log)
392- log->release();
393-
394- if (mysql)
395- mysql_close(mysql);
396- }
397-
398- MSTrans *log;
399- MYSQL *mysql;
400- uint32_t count;
401- uint32_t myActivity;
402-
403- bool stopit;
404- bool finished;
405-
406- virtual bool doWork() {return true;}
407-};
408-
409-//------------------------------------------------
410-class TransTestWriterThread : public TransTestThread {
411-public:
412- TransTestWriterThread():TransTestThread() {}
413-
414- uint32_t tab_id;
415- FILE *myLog;
416-
417- void generate_records();
418- bool doWork()
419- {
420- generate_records();
421- finished = true;
422- return true;
423- }
424-
425- static TransTestWriterThread *newTransTestWriterThread(uint32_t id)
426- {
427- TransTestWriterThread *tt;
428- enter_();
429-
430-
431- new_(tt, TransTestWriterThread());
432-
433- char name[32];
434- sprintf(name, "write_%d.log", id);
435- if (recreate)
436- tt->myLog = fopen(name, "w+");
437- else {
438- tt->myLog = fopen(name, "a+");
439- fprintf(tt->myLog, "====================================================\n");
440- }
441-
442- tt->tab_id = id ;
443- tt->mysql = new_connection(false);
444- tt->log = trans_log;
445- trans_log->retain();
446-
447- return_(tt);
448- }
449-
450-
451-};
452-
453-//------------------------------------------------
454-class TransTestReaderThread : public TransTestThread {
455-public:
456- TransTestReaderThread():TransTestThread(){}
457-
458- bool recovering;
459- void processTransactionLog();
460- bool doWork()
461- {
462- processTransactionLog();
463- return true;
464- }
465-
466- static TransTestReaderThread *newTransTestReaderThread(MSTrans *log)
467- {
468- TransTestReaderThread *tt;
469- enter_();
470-
471- new_(tt, TransTestReaderThread());
472- tt->mysql = new_connection(false);
473- tt->log = log;
474- tt->log->retain();
475-
476- tt->log->txn_SetReader(tt); // The reader daemon is passed in unreferenced.
477- tt->recovering = false;
478- return_(tt);
479- }
480-
481- bool rec_found(uint64_t id, uint32_t tab_id)
482- {
483- char stmt[100];
484- MYSQL_RES *results = NULL;
485- bool found;
486-
487- sprintf(stmt, "SELECT blob_ref FROM "LOG_TABLE" WHERE blob_ref = %"PRIu64" AND tab_id = %"PRIu32"", id, tab_id);
488- if (mysql_query(mysql, stmt)) {
489- printf( "MySQL ERROR: %d \"%s\" line %d\n", mysql_errno(mysql), mysql_error(mysql), __LINE__);
490- printf("%s\n", stmt);
491- exit(1);
492- }
493-
494-
495- results = mysql_store_result(mysql);
496- if (!results){
497- printf( "MySQL ERROR: %d \"%s\" line %d\n", mysql_errno(mysql), mysql_error(mysql), __LINE__);
498- exit(1);
499- }
500-
501- found = (mysql_num_rows(results) == 1);
502- mysql_free_result(results);
503-
504- return found;
505-
506- }
507-
508-
509-};
510-
511-TransTestReaderThread *TransReader;
512-//------------------------------------------------
513-static void report_mysql_error(MYSQL *mysql, int line, const char *msg)
514-{
515- printf( "MySQL ERROR: %d \"%s\" line %d\n", mysql_errno(mysql), mysql_error(mysql), line);
516- if (msg)
517- printf("%s\n", msg);
518- exit(1);
519-}
520-
521-
522-//------------------------------------------------
523-static MYSQL *new_connection(bool check_for_db)
524-{
525- MYSQL *mysql;
526-
527- mysql = mysql_init(NULL);
528- if (!mysql) {
529- printf( "mysql_init() failed.\n");
530- exit(1);
531- }
532-
533- if (mysql_real_connect(mysql, host, user_name, user_passwd, NULL, port, NULL, 0) == NULL)
534- report_mysql_error(mysql, __LINE__, "mysql_real_connect()");
535-
536- if (check_for_db) {
537- MYSQL_RES *results = NULL;
538-
539- if (mysql_query(mysql, "show databases like \"" TEST_DATABASE_NAME "\""))
540- report_mysql_error(mysql, __LINE__, "show databases like \"" TEST_DATABASE_NAME "\"");
541-
542- results = mysql_store_result(mysql);
543- if (!results)
544- report_mysql_error(mysql, __LINE__, "mysql_store_result()");
545-
546-
547- if (mysql_num_rows(results) != 1) {
548- if (mysql_query(mysql, "create database " TEST_DATABASE_NAME ))
549- report_mysql_error(mysql, __LINE__, "create database " TEST_DATABASE_NAME );
550- }
551- mysql_free_result(results);
552- }
553-
554- if (mysql_query(mysql, "use " TEST_DATABASE_NAME ))
555- report_mysql_error(mysql, __LINE__, "use " TEST_DATABASE_NAME );
556-
557- return mysql;
558-}
559-
560-//------------------------------------------------
561-static void init_database(MYSQL *mysql, int cnt)
562-{
563- char stmt[1024];
564-
565- unlink("ms-trans-log.dat");
566- mysql_query(mysql, "drop table if exists " LOG_TABLE ";");
567-
568- if (mysql_query(mysql, "create table " LOG_TABLE CREATE_TABLE_BODY ";")){
569- printf( "MySQL ERROR: %d \"%s\" line %d\n", mysql_errno(mysql), mysql_error(mysql), __LINE__);
570- exit(1);
571- }
572-
573- while (cnt) {
574- sprintf(stmt, "drop table if exists " REF_TABLE ";", cnt);
575- mysql_query(mysql, stmt);
576- sprintf(stmt, "create table " REF_TABLE CREATE_TABLE_BODY ";", cnt);
577- if (mysql_query(mysql, stmt)){
578- printf( "MySQL ERROR: %d \"%s\" line %d\n", mysql_errno(mysql), mysql_error(mysql), __LINE__);
579- exit(1);
580- }
581- cnt--;
582- }
583-}
584-
585-
586-//------------------------------------------------
587-static void display_help(const char *app)
588-{
589- printf("\nUsage:\n");
590- printf("%s -help | -r [-t<num_threads>] | -d | [-n] [-sc <cache_size>] [-sl <log_size>] [-c <crash_site>] [-t<num_threads>] [<timeout>]\n\n", app);
591-
592- printf("-r: Test recovery after a crash or shutdown.\n");
593- printf("-d: Dump the transaction log.\n");
594- printf("-n: Recreate the tables and recovery log.\n");
595- printf("-c <crash_site>: Crash at this location rather than shutting down. Max = %d\n", MAX_CRASH_POINT+1);
596- printf("-t<num_threads>: The number of writer threads to use, default is %d.\n", num_threads);
597- //printf("-r<rate>: The number af records to be inserted per second, default is %d.\n", rate);
598- printf("<timeout>: The number seconds the test should run before shuttingdown or crashing, default is %d.\n\n", timeout);
599- exit(1);
600-}
601-
602-//---------------------------------
603-static void process_args(int argc, const char * argv[])
604-{
605- if (argc < 2)
606- return;
607-
608- for (int i = 1; i < argc; ) {
609- if ( argv[i][0] != '-') { // Must be timeout
610- timeout = atoi(argv[i]);
611- i++;
612- if ((i != argc) || !timeout)
613- display_help(argv[0]);
614- } else {
615- switch (argv[i][1]) {
616- case 'h':
617- display_help(argv[0]);
618- break;
619-
620- case 'r':
621- if (argc > 4 || argv[i][2])
622- display_help(argv[0]);
623- revover_only = true;
624- i++;
625- break;
626-
627- case 'd':
628- if (argc != 2 || argv[i][2])
629- display_help(argv[0]);
630- dump_log = true;
631- i++;
632- break;
633-
634- case 'n':
635- if (argv[i][2])
636- display_help(argv[0]);
637- recreate = true;
638- i++;
639- break;
640-
641- case 'c':
642- if (argv[i][2])
643- display_help(argv[0]);
644- i++;
645- crash_site = atoi(argv[i]);
646- if (crash_site == (MAX_CRASH_POINT + 1))
647- overflow_crash = true;
648- else if ((!crash_site) || (crash_site > MAX_CRASH_POINT))
649- display_help(argv[0]);
650- i++;
651- break;
652-
653- case 's': {
654- uint32_t size;
655-
656- size = atol(argv[i+1]);
657- if (!size)
658- display_help(argv[0]);
659-
660- if (argv[i][2] == 'c')
661- cache_size = size;
662- else if (argv[i][2] == 'l')
663- log_size = size;
664- else
665- display_help(argv[0]);
666-
667- i+=2;
668- }
669- break;
670-
671- case 't':
672- if (argv[i][2])
673- display_help(argv[0]);
674- i++;
675- num_threads = atoi(argv[i]);
676- if (!num_threads)
677- display_help(argv[0]);
678- i++;
679- break;
680-/*
681- case 'r':
682- i++;
683- rate = atoi(argv[i]);
684- if (!rate)
685- display_help(argv[0]);
686- i++;
687- break;
688-*/
689- default:
690- display_help(argv[0]);
691- }
692-
693- }
694- }
695-}
696-
697-//---------------------------------
698-static void init_env()
699-{
700- cs_init_memory();
701- CSThread::startUp();
702- if (!(main_thread = CSThread::newCSThread())) {
703- CSException::logOSError(CS_CONTEXT, ENOMEM);
704- exit(1);
705- }
706-
707- CSThread::setSelf(main_thread);
708-
709- enter_();
710- try_(a) {
711- trans_log = MSTrans::txn_NewMSTrans("./ms-trans-log.dat", /*dump_log*/ true);
712- new_(thread_list, CSThreadList());
713- }
714- catch_(a) {
715- self->logException();
716- CSThread::shutDown();
717- exit(1);
718- }
719- cont_(a);
720-
721-}
722-//---------------------------------
723-static void deinit_env()
724-{
725- if (thread_list) {
726- thread_list->release();
727- thread_list = NULL;
728- }
729-
730- if (trans_log) {
731- trans_log->release();
732- trans_log = NULL;
733- }
734-
735- if (main_thread) {
736- main_thread->release();
737- main_thread = NULL;
738- }
739-
740- CSThread::shutDown();
741- cs_exit_memory();
742-}
743-//---------------------------------
744-static bool verify_database(MYSQL *mysql)
745-{
746- MYSQL_RES **r_results, *l_results = NULL;
747- MYSQL_ROW r_record, l_record;
748- bool ok = false;
749- int i, log_row_cnt, ref_row_cnt = 0, tab_id;
750- char stmt[1024];
751-
752- r_results = (MYSQL_RES **) malloc(num_threads * sizeof(MYSQL_RES *));
753-
754- if (mysql_query(mysql, "select * from "LOG_TABLE" where committed = 0 order by blob_ref"))
755- report_mysql_error(mysql, __LINE__, "select * from "LOG_TABLE" order by blob_ref");
756-
757- l_results = mysql_store_result(mysql);
758- if (!l_results)
759- report_mysql_error(mysql, __LINE__, "mysql_store_result()");
760-
761- log_row_cnt = mysql_num_rows(l_results);
762- mysql_free_result(l_results);
763- if (log_row_cnt)
764- printf("Uncommitted references: %d\n", log_row_cnt);
765-
766- //---------
767- for (i =0; i < num_threads; i++) {
768- sprintf(stmt, "select * from "REF_TABLE" order by blob_ref", i+1);
769- if (mysql_query(mysql, stmt))
770- report_mysql_error(mysql, __LINE__, stmt);
771-
772- r_results[i] = mysql_store_result(mysql);
773- if (!r_results)
774- report_mysql_error(mysql, __LINE__, "mysql_store_result()");
775-
776- ref_row_cnt += mysql_num_rows(r_results[i]);
777- }
778- //---------
779- if (mysql_query(mysql, "select * from "LOG_TABLE" order by blob_ref"))
780- report_mysql_error(mysql, __LINE__, "select * from "LOG_TABLE" order by blob_ref");
781-
782- l_results = mysql_store_result(mysql);
783- if (!l_results)
784- report_mysql_error(mysql, __LINE__, "mysql_store_result()");
785-
786- log_row_cnt = mysql_num_rows(l_results);
787-
788- if (log_row_cnt != ref_row_cnt) {
789- if (ref_row_cnt > log_row_cnt) {
790- printf("verify_database() Failed: row count doesn't match: log_row_cnt(%d) != ref_row_cnt(%d)\n", log_row_cnt, ref_row_cnt);
791- goto done;
792- }
793-
794- printf("verify_database() Warnning: row count doesn't match: log_row_cnt(%d) != ref_row_cnt(%d)\n", log_row_cnt, ref_row_cnt);
795- printf("Possible unreferenced BLOBs\n");
796- }
797-
798- if (log_row_cnt == ref_row_cnt) {
799- for ( i = 0; i < log_row_cnt; i++) {
800- l_record = mysql_fetch_row(l_results);
801- tab_id = atol(l_record[1]);
802- r_record = mysql_fetch_row(r_results[tab_id-1]);
803- if ((atol(l_record[0]) != atol(r_record[0])) ||
804- (atol(l_record[1]) != atol(r_record[1])) ||
805- (atol(l_record[2]) != atol(r_record[2]))) {
806-
807- printf("verify_database() Failed: in row %d, tab_id %d\n", i+1, tab_id);
808- printf("field 1: %d =? %d\n", atol(l_record[0]), atol(r_record[0]));
809- printf("field 2: %d =? %d\n", atol(l_record[1]), atol(r_record[1]));
810- printf("field 3: %d =? %d\n", atol(l_record[2]), atol(r_record[2]));
811- goto done;
812- }
813-
814- }
815- } else { // The important thing is that there are no BLOBs in the ref tabels that are not in the log table.
816-
817- for (i =0; i < num_threads; i++) {
818- mysql_free_result(r_results[i]);
819-
820- sprintf(stmt, "select * from "REF_TABLE" where blob_ref not in (select blob_ref from TransTest.translog where tab_id = %d)", i+1, i+1);
821- if (mysql_query(mysql, stmt))
822- report_mysql_error(mysql, __LINE__, stmt);
823-
824- r_results[i] = mysql_store_result(mysql);
825- if (!r_results)
826- report_mysql_error(mysql, __LINE__, "mysql_store_result()");
827-
828- if (mysql_num_rows(r_results[i])) {
829- printf("verify_database() Failed, Missing BLOBs: %s\n", stmt);
830- goto done;
831- }
832- }
833- }
834-
835- printf("verify_database() OK.\n");
836- ok = true;
837-
838- done:
839-
840- for (i =0; i < num_threads; i++) {
841- mysql_free_result(r_results[i]);
842- }
843- free(r_results);
844-
845- mysql_free_result(l_results);
846-
847-#ifdef DEBUG
848- if (!ok) {
849- trans_log->txn_DumpLog("trace.log");
850- }
851-#endif
852- return ok;
853-}
854-
855-//------------------------------------------------
856-void TransTestReaderThread::processTransactionLog()
857-{
858- MSTransRec rec = {0};
859- MS_TxnState state;
860- char stmt[1024];
861- uint32_t last_tid = 0;
862- enter_();
863-
864- // Read in transactions from the log and update
865- // the database table based on them.
866-
867- try_(a) {
868- while (!myMustQuit && !stopit) {
869- // This will sleep while waiting for the next
870- // completed transaction.
871- log->txn_GetNextTransaction(&rec, &state);
872- if (myMustQuit)
873- break;
874-
875- myActivity++;
876-#ifdef CHECK_TIDS
877- if (num_threads == 1) {
878- ASSERT( ((last_tid + 1) == rec.tr_id) || (last_tid == rec.tr_id) || !last_tid);
879- last_tid = rec.tr_id;
880- }
881-#endif
882- if (!recovering)
883- count++;
884-
885- switch (TRANS_TYPE(rec.tr_type)) {
886- case MS_ReferenceTxn:
887- case MS_DereferenceTxn:
888- case MS_RollBackTxn:
889- case MS_CommitTxn:
890- case MS_RecoveredTxn:
891- break;
892- default:
893- printf("Unexpected transaction type: %d\n", rec.tr_type);
894- exit(1);
895- }
896-
897- if (state == MS_Committed){
898- // Dereferences are applied when the transaction is commited.
899- // References are applied imediatly and removed if the transaction is rolled back.
900- if (TRANS_TYPE(rec.tr_type) == MS_DereferenceTxn) {
901- sprintf(stmt, "DELETE FROM "LOG_TABLE" WHERE blob_ref = %"PRIu64" AND tab_id = %d AND blob_id = %"PRIu64"", rec.tr_blob_ref_id, rec.tr_tab_id, rec.tr_blob_id);
902- if (mysql_query(mysql, stmt))
903- report_mysql_error(mysql, __LINE__, stmt);
904- } else if (TRANS_TYPE(rec.tr_type) == MS_ReferenceTxn) {
905- sprintf(stmt, "UPDATE "LOG_TABLE" SET committed = 1 WHERE blob_ref = %"PRIu64" AND tab_id = %d AND blob_id = %"PRIu64"", rec.tr_blob_ref_id, rec.tr_tab_id, rec.tr_blob_id);
906- if (mysql_query(mysql, stmt))
907- report_mysql_error(mysql, __LINE__, stmt);
908- }
909- } else if (state == MS_RolledBack) {
910- //printf("ROLLBACK!\n");
911- if (TRANS_TYPE(rec.tr_type) == MS_ReferenceTxn) {
912- sprintf(stmt, "DELETE FROM "LOG_TABLE" WHERE blob_ref = %"PRIu64" AND tab_id = %d AND blob_id = %"PRIu64"", rec.tr_blob_ref_id, rec.tr_tab_id, rec.tr_blob_id);
913- if (mysql_query(mysql, stmt))
914- report_mysql_error(mysql, __LINE__, stmt);
915- }
916- } else if (state == MS_Recovered) {
917- printf("Recovered transaction being ignored:\n");
918- printf("blob_ref = %"PRIu64", tab_id = %d, blob_id = %"PRIu64"\n\n", rec.tr_blob_ref_id, rec.tr_tab_id, rec.tr_blob_id);
919- } else {
920- printf("Unexpected transaction state: %d\n", state);
921- exit(1);
922- }
923-
924-
925- }
926- }
927- catch_(a) {
928- self->logException();
929- printf("\n\n!!!!!!!! THE TRANSACTION LOG READER DIED! !!!!!!!!!!!\n\n");
930- if (!myMustQuit && !stopit)
931- exit(1);
932- }
933- cont_(a);
934- printf("The transaction log reader shutting down.\n");
935- exit_();
936-}
937-
938-//------------------------------------------------
939-void TransTestWriterThread::generate_records()
940-{
941-
942- MS_Txn txn_type;
943- uint64_t blob_id;
944- uint64_t blob_ref_id;
945- int tsize, i;
946- bool do_delete;
947-
948- char stmt[1024];
949- enter_();
950-
951- try_(a) {
952- while (!myMustQuit && !stopit) {
953-
954- myActivity++;
955- usleep(nap_time); // Give up a bit of time
956- if (myMustQuit || stopit)
957- break;
958-
959- tsize = rand() % max_transaction;
960-
961- if (mysql_autocommit(mysql, 0))
962- report_mysql_error(mysql, __LINE__, "mysql_autocommit()");
963-
964- i = 0;
965- do {
966- do_delete = ((rand() %2) == 0);
967-
968- // decide if this is an insert or delete
969- if (do_delete) {
970- MYSQL_RES *results = NULL;
971- MYSQL_ROW record;
972- int cnt;
973-
974- // If we are deleting then randomly select a record to delete
975- // and delete it.
976-
977- txn_type = MS_DereferenceTxn;
978-
979- sprintf(stmt, "select * from "REF_TABLE, tab_id);
980- if (mysql_query(mysql, stmt))
981- report_mysql_error(mysql, __LINE__, stmt);
982-
983- results = mysql_store_result(mysql);
984- if (!results)
985- report_mysql_error(mysql, __LINE__, "mysql_store_result()");
986-
987- cnt = mysql_num_rows(results);
988- if (!cnt)
989- do_delete = false; // There is nothing to delete
990- else {
991- mysql_data_seek(results, rand()%cnt);
992- record = mysql_fetch_row(results);
993-
994- blob_ref_id = atol(record[0]);
995- blob_id = atol(record[2]);
996-
997- sprintf(stmt, "DELETE FROM "REF_TABLE" WHERE blob_ref = %"PRIu64" AND blob_id = %"PRIu64"", tab_id, blob_ref_id, blob_id);
998- if (mysql_query(mysql, stmt))
999- report_mysql_error(mysql, __LINE__, stmt);
1000-
1001- if (mysql_affected_rows(mysql) == 0)
1002- do_delete = false; // Another thread must have deleted the row first.
1003- else
1004- fprintf(myLog, "DELETE %"PRIu64" %"PRIu64"\n", blob_ref_id, blob_id);
1005- }
1006-
1007- mysql_free_result(results);
1008- }
1009-
1010- if (!do_delete) {
1011- blob_id = self->myTID; // Assign the tid as the blob id to help with debugging.
1012- txn_type = MS_ReferenceTxn;
1013-
1014- sprintf(stmt, "INSERT INTO "REF_TABLE" VALUES( NULL, %d, %"PRIu64", 0)", tab_id, tab_id, blob_id);
1015- if (mysql_query(mysql, stmt))
1016- report_mysql_error(mysql, __LINE__, stmt);
1017-
1018- blob_ref_id = mysql_insert_id(mysql);
1019- if (!blob_ref_id)
1020- report_mysql_error(mysql, __LINE__, "mysql_insert_id() returned 0");
1021-
1022- fprintf(myLog, "INSERT %"PRIu64" %"PRIu64"\n", blob_ref_id, blob_id);
1023- // Apply the blob reference now. This will be undone if the transaction is rolled back.
1024- sprintf(stmt, "INSERT INTO "LOG_TABLE" VALUES(%"PRIu64", %d, %"PRIu64", 0)", blob_ref_id, tab_id, blob_id);
1025- if (mysql_query(mysql, stmt))
1026- report_mysql_error(mysql, __LINE__, stmt);
1027- }
1028-
1029- i++;
1030- count++;
1031- if (i >= tsize) { //Commit the database transaction before the log transaction.
1032- bool rollback;
1033-
1034- rollback = ((tsize > 0) && ((rand() % 1000) == 0));
1035- if (rollback) {
1036- printf("Rollback\n");
1037- if (mysql_rollback(mysql)) // commit the staement to the database,
1038- report_mysql_error(mysql, __LINE__, "mysql_rollback()");
1039- fprintf(myLog, "Rollback %"PRIu32"\n", self->myTID);
1040- log->txn_LogTransaction(MS_RollBackTxn);
1041- } else {
1042- if (mysql_commit(mysql)) // commit the staement to the database,
1043- report_mysql_error(mysql, __LINE__, "mysql_commit()");
1044- fprintf(myLog, "Commit %"PRIu32"\n", self->myTID);
1045- log->txn_LogTransaction(txn_type, true, A_DB_ID, tab_id, blob_id, blob_ref_id);
1046- }
1047- } else
1048- log->txn_LogTransaction(txn_type, false, A_DB_ID, tab_id, blob_id, blob_ref_id);
1049-
1050- } while ( i < tsize);
1051-
1052- }
1053- }
1054-
1055- catch_(a) {
1056- self->logException();
1057- printf("\n\nA writer thread for table %d died! \n\n", tab_id);
1058- if (i == tsize) {
1059- printf(" It is possible that the last %d operations on table %d were committed to the database but not to the log.\n", tsize, tab_id);
1060- }
1061- if (!myMustQuit && !stopit)
1062- exit(1);
1063- }
1064- cont_(a);
1065- printf("Writer thread for table %d is shutting down.\n", tab_id);
1066- exit_();
1067-}
1068-
1069-// SELECT * FROM TransTest.translog where blob_ref not in (select blob_ref from TransTest.transref)
1070-// SELECT * FROM TransTest.transref_1 where blob_ref not in (select blob_ref from TransTest.translog where tab_id = 1)
1071-// SELECT * FROM TransTest.translog where tab_id = 1 AND blob_ref not in (select blob_ref from TransTest.transref_1)
1072-// select count(*) from TransTest.translog where committed = 1
1073-//---------------------------------
1074-int main (int argc, const char * argv[])
1075-{
1076- MYSQL *mysql;
1077- TransTestWriterThread **writer = NULL;
1078- int rtc = 1;
1079-
1080- process_args(argc, argv);
1081-
1082- mysql = new_connection(true);
1083-
1084- if (recreate)
1085- init_database(mysql, num_threads);
1086-
1087- init_env();
1088- enter_();
1089-
1090- if (dump_log) {
1091- printf("LOG dumped\n");
1092- exit(1);
1093- }
1094-
1095- TransReader = TransTestReaderThread::newTransTestReaderThread(trans_log);
1096- push_(TransReader);
1097- TransReader->recovering = true;
1098- TransReader->start();
1099-
1100- // wait until the recovery is complete.
1101- while (trans_log->txn_GetNumRecords())
1102- usleep(100);
1103-
1104- TransReader->recovering = false;
1105-
1106- if (log_size)
1107- trans_log->txn_SetLogSize(log_size);
1108-
1109- if (cache_size)
1110- trans_log->txn_SetCacheSize(cache_size);
1111-
1112- if (revover_only) {
1113- TransReader->stopit = true;
1114- if (verify_database(mysql))
1115- rtc = 0;
1116- goto done;
1117- }
1118-
1119- try_(a) {
1120- writer = (TransTestWriterThread **) cs_malloc(num_threads * sizeof(TransTestWriterThread *));
1121- for (int i = 0; i < num_threads; i++) {
1122- TransTestWriterThread *wt = TransTestWriterThread::newTransTestWriterThread(i+1);
1123- wt->start();
1124- writer[i] = wt;
1125- }
1126-
1127- printf("Timeout: %d seconds\n", timeout);
1128- timeout += time(NULL);
1129- int header = 0;
1130- while (timeout > time(NULL)) {
1131- MSTransStatsRec stats;
1132- self->sleep(1000);
1133- trans_log->txn_GetStats(&stats);
1134-
1135-
1136- if (!(header%20)) {
1137- for (int i = 0; i < num_threads; i++) {
1138- if (writer[i]->myActivity == 0) {
1139- printf("Writer thread %d HUNG!!!\n", i);
1140- }
1141- writer[i]->myActivity = 0;
1142- }
1143-
1144- if (TransReader->myActivity == 0) {
1145- printf("Reader thread HUNG!!!\n");
1146- }
1147- TransReader->myActivity = 0;
1148-
1149- printf("%s | %s | %s | %s | %s | %s | %s | %s\n", "LogSize", "Full", "MaxSize", "Overflows", "Overflowing", "CacheSize", "Cache Used", "Cache Hit");
1150- }
1151- header++;
1152- //printf("Writes: %d \t\t Reads: %d \t%d \t start: %"PRIu64"\t\t eol:%"PRIu64"\n", count, TransReader->count, count - TransReader->count, trans_log->txn_Start, trans_log->txn_EOL);
1153- printf("%7llu | %3d%% | %7llu | %9d | %11s | %9d | %9d%% | %9d%%\n",// | \t\t\t%"PRIu64" \t%"PRIu64"\n",
1154- stats.ts_LogSize,
1155- stats.ts_PercentFull,
1156- stats.ts_MaxSize,
1157- stats.ts_OverflowCount,
1158- (stats.ts_IsOverflowing)?"Over Flow": " --- ",
1159- stats.ts_TransCacheSize,
1160- stats.ts_PercentTransCacheUsed,
1161- stats.ts_PercentCacheHit//, trans_log->txn_Start, trans_log->txn_EOL
1162- );
1163-
1164- if (stats.ts_IsOverflowing && overflow_crash) {
1165- printf("Simulating crash while in overflow\n");
1166- exit(1);
1167- }
1168- }
1169-
1170-#ifdef CRASH_TEST
1171- if (crash_site) {
1172- printf("Crashing at crash site %d\n", crash_site);
1173- trans_test_crash_point = crash_site;
1174- // set the crash site and wait to die.
1175- while(1)
1176- self->sleep(1000);
1177- }
1178-#endif
1179-
1180- printf("Shutting down the writer threads:\n");
1181- for (int i = 0; i < num_threads; i++) {
1182- writer[i]->stopit = true;
1183- }
1184-
1185- TransReader->stopit = true;
1186- // Give the writers a chance to shutdown by themselves.
1187- int cnt = 100;
1188- while (cnt) {
1189- int i;
1190- for (i = 0; i < num_threads && writer[i]->finished; i++);
1191- if (i == num_threads && TransReader->finished)
1192- break;
1193- self->sleep(10);
1194- cnt--;
1195- }
1196-
1197- for (int i = 0; i < num_threads; i++) {
1198- writer[i]->stop();
1199- }
1200-
1201- }
1202- rtc = 0;
1203- catch_(a) {
1204- printf("Main thread abort.\n");
1205- self->logException();
1206- }
1207- cont_(a);
1208- if (writer) {
1209- for (int i = 0; i < num_threads; i++) {
1210- writer[i]->stop();
1211- writer[i]->release();
1212- }
1213- cs_free(writer);
1214- }
1215-
1216-done:
1217- TransReader->stop();
1218- release_(TransReader);
1219-
1220- outer_();
1221-
1222- thread_list->stopAllThreads();
1223- deinit_env();
1224- mysql_close(mysql);
1225- exit(rtc);
1226-}
1227-
1228-#endif // UNIT_TEST
1229
1230=== removed file 'plugin/pbms/src/alias_ms.cc'
1231--- plugin/pbms/src/alias_ms.cc 2010-12-18 04:43:40 +0000
1232+++ plugin/pbms/src/alias_ms.cc 1970-01-01 00:00:00 +0000
1233@@ -1,1092 +0,0 @@
1234-/* Copyright (C) 2008 PrimeBase Technologies GmbH, Germany
1235- *
1236- * PrimeBase Media Stream for MySQL
1237- *
1238- * This program is free software; you can redistribute it and/or modify
1239- * it under the terms of the GNU General Public License as published by
1240- * the Free Software Foundation; either version 2 of the License, or
1241- * (at your option) any later version.
1242- *
1243- * This program is distributed in the hope that it will be useful,
1244- * but WITHOUT ANY WARRANTY; without even the implied warranty of
1245- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
1246- * GNU General Public License for more details.
1247- *
1248- * You should have received a copy of the GNU General Public License
1249- * along with this program; if not, write to the Free Software
1250- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
1251- *
1252- * Barry Leslie
1253- *
1254- * 2008-12-30
1255- *
1256- * H&G2JCtL
1257- *
1258- * BLOB alias index.
1259- *
1260- */
1261-
1262-#ifdef HAVE_ALIAS_SUPPORT
1263-#include "cslib/CSConfig.h"
1264-
1265-#include "string.h"
1266-
1267-#ifdef DRIZZLED
1268-#include <drizzled/common.h>
1269-#endif
1270-
1271-#include "cslib/CSGlobal.h"
1272-#include "cslib/CSLog.h"
1273-#include "cslib/CSStrUtil.h"
1274-#include "cslib/CSFile.h"
1275-#include "system_table_ms.h"
1276-#include "database_ms.h"
1277-
1278-#include "alias_ms.h"
1279-
1280-
1281-
1282-//------------------------
1283-MSAlias::~MSAlias()
1284-{
1285- enter_();
1286-
1287- ASSERT(iClosing);
1288- ASSERT(iPoolSysTables.getSize() == 0);
1289-
1290-
1291- if (iFilePath) {
1292-
1293- if (iDelete)
1294- iFilePath->removeFile();
1295-
1296- iFilePath->release();
1297- }
1298-
1299- if (iFileShare)
1300- iFileShare->release();
1301-
1302- exit_();
1303-}
1304-
1305-//------------------------
1306-MSAlias::MSAlias(MSDatabase *db_noref)
1307-{
1308- iClosing = false;
1309- iDelete = false;
1310- iDatabase_br = db_noref;
1311- iFilePath = NULL;
1312- iFileShare = NULL;
1313-}
1314-
1315-//------------------------
1316-void MSAlias::ma_close()
1317-{
1318- enter_();
1319-
1320- iClosing = true;
1321- if (iFileShare)
1322- iFileShare->close();
1323- iPoolSysTables.clear();
1324- exit_();
1325-}
1326-
1327-//------------------------
1328-// Compress the index bucket chain and free unused buckets.
1329-void MSAlias::MSAliasCompress(CSFile *fa, CSSortedList *freeList, MSABucketLinkedList *bucketChain)
1330-{
1331- // For now I will just remove empty buckets.
1332- // Later this function should also compress the records also
1333- // thus making the searches faster and freeing up more space.
1334- MSABucketInfo *b_info, *next;
1335-
1336- b_info = bucketChain->getFront();
1337- while (b_info) {
1338- next = b_info->getNextLink();
1339- if (b_info->getSize() == 0) {
1340- bucketChain->remove(RETAIN(b_info));
1341- freeList->add(b_info);
1342- }
1343- b_info = next;
1344- }
1345-
1346-}
1347-
1348-//------------------------
1349-void MSAlias::MSAliasLoad()
1350-{
1351- CSFile *fa = NULL;
1352- CSSortedList freeList;
1353- off64_t fileSize;
1354-
1355- enter_();
1356-
1357- fa = CSFile::newFile(RETAIN(iFilePath));
1358- push_(fa);
1359-
1360- MSAliasHeadRec header;
1361- uint64_t free_list_offset;
1362- fa->open(CSFile::DEFAULT);
1363- fa->read(&header, 0, sizeof(header), sizeof(header));
1364-
1365- /* Check the file header: */
1366- if (CS_GET_DISK_4(header.ah_magic_4) != MS_ALIAS_FILE_MAGIC)
1367- CSException::throwFileError(CS_CONTEXT, iFilePath->getCString(), CS_ERR_BAD_HEADER_MAGIC);
1368- if (CS_GET_DISK_2(header.ah_version_2) != MS_ALIAS_FILE_VERSION)
1369- CSException::throwFileError(CS_CONTEXT, iFilePath->getCString(), CS_ERR_VERSION_TOO_NEW);
1370-
1371- free_list_offset = CS_GET_DISK_8(header.ah_free_list_8);
1372-
1373- fileSize = CS_GET_DISK_8(header.ah_file_size_8);
1374-
1375- // Do some sanity checks.
1376- if (CS_GET_DISK_2(header.ah_head_size_2) != sizeof(header))
1377- CSException::throwFileError(CS_CONTEXT, iFilePath->getCString(), CS_ERR_BAD_FILE_HEADER);
1378-
1379- if (CS_GET_DISK_2(header.ah_num_buckets_2) != BUCKET_LIST_SIZE)
1380- CSException::throwFileError(CS_CONTEXT, iFilePath->getCString(), CS_ERR_BAD_FILE_HEADER);
1381-
1382- if (CS_GET_DISK_4(header.ah_bucket_size_4) != NUM_RECORDS_PER_BUCKET)
1383- CSException::throwFileError(CS_CONTEXT, iFilePath->getCString(), CS_ERR_BAD_FILE_HEADER);
1384-
1385- if (fileSize != fa->getEOF())
1386- CSException::throwFileError(CS_CONTEXT, iFilePath->getCString(), CS_ERR_BAD_FILE_HEADER);
1387-
1388- // Load the bucket headers into RAM
1389- MSADiskBucketHeadRec bucketHead = {0};
1390- uint64_t offset, start_offset;
1391-
1392- // Fist load the free list:
1393- if (free_list_offset) {
1394- start_offset = offset = free_list_offset;
1395- do {
1396- fa->read(&bucketHead, offset, sizeof(MSADiskBucketHeadRec), sizeof(MSADiskBucketHeadRec));
1397- freeList.add(MSABucketInfo::newMSABucketInfo(offset));
1398- offset = CS_GET_DISK_8(bucketHead.ab_next_bucket_8);
1399- } while (offset != start_offset);
1400-
1401- }
1402- for (uint32_t i = 0; i < BUCKET_LIST_SIZE; i++) {
1403- uint64_t used, total_space;
1404- MSABucketLinkedList *bucketChain = &(iFileShare->msa_buckets[i]);
1405-
1406- start_offset = offset = sizeof(header) + i * sizeof(MSADiskBucketRec);
1407- used = total_space = 0;
1408- do {
1409- uint32_t num, end_of_records;
1410-
1411- fa->read(&bucketHead, offset, sizeof(MSADiskBucketHeadRec), sizeof(MSADiskBucketHeadRec));
1412- num = CS_GET_DISK_4(bucketHead.ab_num_recs_4);
1413- end_of_records = CS_GET_DISK_4(bucketHead.ab_eor_rec_4);
1414- total_space += NUM_RECORDS_PER_BUCKET;
1415- used += num;
1416- bucketChain->addFront(MSABucketInfo::newMSABucketInfo(offset, num, end_of_records));
1417- offset = CS_GET_DISK_8(bucketHead.ab_next_bucket_8);
1418-
1419- } while (offset != start_offset);
1420-
1421- // Pack the index if required
1422- if (((total_space - used) / NUM_RECORDS_PER_BUCKET) > 1)
1423- MSAliasCompress(fa, &freeList, bucketChain);
1424-
1425- }
1426-
1427- // If there are free buckets try to free up some disk
1428- // space or add them to a free list to be reused later.
1429- if (freeList.getSize()) {
1430- uint64_t last_bucket = fileSize - sizeof(MSADiskBucketRec);
1431- MSABucketInfo *rec;
1432- bool reduce = false;
1433-
1434- // Search for freed buckets at the end of the file
1435- // so that they can be released and the file
1436- // shrunk.
1437- //
1438- // The free list has been sorted so that buckets
1439- // with the highest file offset are first.
1440- do {
1441- rec = (MSABucketInfo*) freeList.itemAt(0);
1442- if (rec->bi_bucket_offset != last_bucket);
1443- break;
1444-
1445- last_bucket -= sizeof(MSADiskBucketRec);
1446- freeList.remove(rec);
1447- reduce = true;
1448- } while (freeList.getSize());
1449-
1450- if (reduce) {
1451- // The file can be reduced in size.
1452- fileSize = last_bucket + sizeof(MSADiskBucketRec);
1453- fa->setEOF(fileSize);
1454- CS_SET_DISK_8(header.ah_file_size_8, fileSize);
1455- fa->write(&header.ah_file_size_8, offsetof(MSAliasHeadRec,ah_file_size_8) , 8);
1456- }
1457-
1458- // Add the empty buckets to the index file's empty bucket list.
1459- memset(&bucketHead, 0, sizeof(bucketHead));
1460- offset = 0;
1461- while (freeList.getSize()) { // Add the empty buckets to the empty_bucket list.
1462- rec = (MSABucketInfo*) freeList.takeItemAt(0);
1463-
1464- // buckets are added to the front of the list.
1465- fa->write(&offset, rec->bi_bucket_offset + offsetof(MSADiskBucketHeadRec,ab_next_bucket_8) , 8);
1466- offset = rec->bi_bucket_offset;
1467- fa->write(&offset, offsetof(MSAliasHeadRec,ah_free_list_8) , 8);
1468-
1469- iFileShare->msa_empty_buckets.addFront(rec);
1470- }
1471- }
1472-
1473- iFileShare->msa_fileSize = fa->getEOF();
1474-
1475- release_(fa);
1476- exit_();
1477-}
1478-
1479-//------------------------
1480-void MSAlias::buildAliasIndex()
1481-{
1482- MSBlobHeadRec blob;
1483- MSRepository *repo;
1484- uint64_t blob_size, fileSize, offset;
1485- uint16_t head_size;
1486- MSAliasFile *afile;
1487- MSAliasRec aliasRec;
1488-
1489- enter_();
1490-
1491- afile = getAliasFile();
1492- frompool_(afile);
1493-
1494- afile->startLoad();
1495-
1496- CSSyncVector *repo_list = iDatabase_br->getRepositoryList();
1497-
1498- // No locking is required since the index is loaded before the database is opened
1499- // and the compactor thread is started.
1500-
1501- for (uint32_t repo_index =0; repo_index<repo_list->size(); repo_index++) {
1502- if ((repo = (MSRepository *) repo_list->get(repo_index))) {
1503- MSRepoFile *repoFile = repo->openRepoFile();
1504- push_(repoFile);
1505- fileSize = repo->getRepoFileSize();
1506- offset = repo->getRepoHeadSize();
1507-
1508- aliasRec.repo_id = repoFile->myRepo->getRepoID();
1509-
1510- while (offset < fileSize) {
1511- if (repoFile->read(&blob, offset, sizeof(MSBlobHeadRec), 0) < sizeof(MSBlobHeadRec))
1512- break;
1513-
1514- if ((CS_GET_DISK_1(blob.rb_status_1) == MS_BLOB_REFERENCED) && CS_GET_DISK_2(blob.rb_alias_offset_2)) {
1515- aliasRec.repo_offset = offset;
1516- aliasRec.alias_hash = CS_GET_DISK_4(blob.rb_alias_hash_4);
1517- addAlias(afile, &aliasRec);
1518- }
1519-
1520- head_size = CS_GET_DISK_2(blob.rb_head_size_2);
1521- blob_size = CS_GET_DISK_6(blob.rb_blob_repo_size_6);
1522- offset += head_size + blob_size;
1523- }
1524-
1525- release_(repoFile);
1526- }
1527- }
1528-
1529- afile->finishLoad();
1530- backtopool_(afile);
1531-
1532- exit_();
1533-}
1534-
1535-//------------------------
1536-void MSAlias::MSAliasBuild()
1537-{
1538- CSFile *fa;
1539- MSAliasHeadRec header = {0};
1540- uint64_t offset, size = sizeof(header) + BUCKET_LIST_SIZE * sizeof(MSADiskBucketRec);
1541- enter_();
1542-
1543- fa = CSFile::newFile(RETAIN(iFilePath));
1544- push_(fa);
1545-
1546- fa->open(CSFile::CREATE | CSFile::TRUNCATE);
1547-
1548- // Create an empty index with 1 empty bucket in each bucket chain.
1549-
1550- CS_SET_DISK_4(header.ah_magic_4, MS_ALIAS_FILE_MAGIC);
1551- CS_SET_DISK_2(header.ah_version_2, MS_ALIAS_FILE_VERSION);
1552-
1553- CS_SET_DISK_2(header.ah_head_size_2, sizeof(header));
1554- CS_SET_DISK_8(header.ah_file_size_8, size);
1555-
1556- CS_SET_DISK_2(header.ah_num_buckets_2, BUCKET_LIST_SIZE);
1557- CS_SET_DISK_2(header.ah_bucket_size_4, NUM_RECORDS_PER_BUCKET);
1558-
1559- fa->setEOF(size); // Grow the file.
1560- fa->write(&header, 0, sizeof(header));
1561-
1562- offset = sizeof(header);
1563-
1564- // Initialize the file bucket chains.
1565- MSADiskBucketHeadRec bucketHead = {0};
1566- for (uint32_t i = 0; i < BUCKET_LIST_SIZE; i++) {
1567- CS_SET_DISK_8(bucketHead.ab_prev_bucket_8, offset);
1568- CS_SET_DISK_8(bucketHead.ab_next_bucket_8, offset);
1569- fa->write(&bucketHead, offset, sizeof(MSADiskBucketHeadRec));
1570- // Add the bucket to the RAM based list.
1571- iFileShare->msa_buckets[i].addFront(MSABucketInfo::newMSABucketInfo(offset));
1572- offset += sizeof(MSADiskBucketRec); // NOTE: MSADiskBucketRec not MSADiskBucketHeadRec
1573- }
1574-
1575- fa->sync();
1576-
1577-
1578-
1579- fa->close();
1580-
1581- release_(fa);
1582-
1583- // Scan through all the BLOBs in the repository and add an entry
1584- // for each blob alias.
1585- buildAliasIndex();
1586-
1587- exit_();
1588-}
1589-
1590-//------------------------
1591-void MSAlias::ma_open(const char *file_name)
1592-{
1593- bool isdir = false;
1594-
1595- enter_();
1596-
1597- iFilePath = CSPath::newPath(RETAIN(iDatabase_br->myDatabasePath), file_name);
1598-
1599-retry:
1600- new_(iFileShare, MSAliasFileShare(RETAIN(iFilePath)));
1601-
1602- if (iFilePath->exists(&isdir)) {
1603- try_(a) {
1604- MSAliasLoad();
1605- }
1606- catch_(a) {
1607- // If an error occurs delete the index and rebuild it.
1608- self->myException.log(NULL);
1609- iFileShare->release();
1610- iFilePath->removeFile();
1611- goto retry;
1612- }
1613- cont_(a);
1614- } else
1615- MSAliasBuild();
1616-
1617-
1618- exit_();
1619-}
1620-
1621-//------------------------
1622-uint32_t MSAlias::hashAlias(const char *ptr)
1623-{
1624- register uint32_t h = 0, g;
1625-
1626- while (*ptr) {
1627- h = (h << 4) + (uint32_t) toupper(*ptr++);
1628- if ((g = (h & 0xF0000000)))
1629- h = (h ^ (g >> 24)) ^ g;
1630- }
1631-
1632- return (h);
1633-}
1634-
1635-//------------------------
1636-void MSAlias::addAlias(MSAliasFile *af, MSAliasRec *rec)
1637-{
1638- MSDiskAliasRec diskRec;
1639- CS_SET_DISK_4(diskRec.ar_repo_id_4, rec->repo_id);
1640- CS_SET_DISK_8(diskRec.ar_offset_8, rec->repo_offset);
1641- CS_SET_DISK_4(diskRec.ar_hash_4, rec->alias_hash);
1642- af->addRec(&diskRec);
1643-
1644-}
1645-
1646-//------------------------
1647-uint32_t MSAlias::addAlias(uint32_t repo_id, uint64_t repo_offset, const char *alias)
1648-{
1649- MSDiskAliasRec diskRec;
1650- uint32_t hash;
1651- uint32_t f_repo_id;
1652- uint64_t f_repo_offset;
1653- bool referenced = false;
1654- enter_();
1655-
1656- hash = hashAlias(alias);
1657-
1658- // Use a lock to make sure that the same alias cannot be added at the same time.
1659- lock_(this);
1660-
1661- MSAliasFile *af = getAliasFile();
1662- frompool_(af);
1663-
1664- if (findBlobByAlias(RETAIN(af), alias, &referenced, &f_repo_id, &f_repo_offset)) {
1665- if ((f_repo_id == repo_id) && (f_repo_offset == repo_offset))
1666- goto done; // Do not treat this as an error.
1667- if (!referenced) {
1668- // If the alias is in use by a non referenced BLOB then delete it.
1669- // This can happen because I allow newly created BLOBs to be accessed
1670- // by their alias even before a reference to the BLOB has been added to
1671- // the database.
1672- af->deleteCurrentRec();
1673- } else {
1674-#ifdef xxDEBUG
1675- CSL.log(self, CSLog::Protocol, "Alias: ");
1676- CSL.log(self, CSLog::Protocol, alias);
1677- CSL.log(self, CSLog::Protocol, "\n");
1678-#endif
1679- CSException::throwException(CS_CONTEXT, MS_ERR_DUPLICATE, "Alias Exists");
1680- }
1681- }
1682-
1683- CS_SET_DISK_4(diskRec.ar_repo_id_4, repo_id);
1684- CS_SET_DISK_8(diskRec.ar_offset_8, repo_offset);
1685- CS_SET_DISK_4(diskRec.ar_hash_4, hash);
1686-
1687- af->addRec(&diskRec);
1688-done:
1689- backtopool_(af);
1690-
1691- unlock_(this);
1692- return_(hash);
1693-}
1694-
1695-//------------------------
1696-void MSAlias::deleteAlias(MSDiskAliasPtr diskRec)
1697-{
1698- enter_();
1699-
1700- MSAliasFile *af = getAliasFile();
1701- frompool_(af);
1702- if (af->findRec(diskRec))
1703- af->deleteCurrentRec();
1704- backtopool_(af);
1705-
1706- exit_();
1707-}
1708-
1709-//------------------------
1710-void MSAlias::deleteAlias(uint32_t repo_id, uint64_t repo_offset, uint32_t alias_hash)
1711-{
1712- MSDiskAliasRec diskRec;
1713-
1714- CS_SET_DISK_4(diskRec.ar_repo_id_4, repo_id);
1715- CS_SET_DISK_8(diskRec.ar_offset_8, repo_offset);
1716- CS_SET_DISK_4(diskRec.ar_hash_4, alias_hash);
1717- deleteAlias(&diskRec);
1718-
1719-}
1720-//------------------------
1721-void MSAlias::resetAlias(uint32_t old_repo_id, uint64_t old_repo_offset, uint32_t alias_hash, uint32_t new_repo_id, uint64_t new_repo_offset)
1722-{
1723- MSDiskAliasRec diskRec;
1724- bool found;
1725- enter_();
1726-
1727- CS_SET_DISK_4(diskRec.ar_repo_id_4, old_repo_id);
1728- CS_SET_DISK_8(diskRec.ar_offset_8, old_repo_offset);
1729- CS_SET_DISK_4(diskRec.ar_hash_4, alias_hash);
1730-
1731- lock_(this);
1732-
1733- MSAliasFile *af = getAliasFile();
1734- frompool_(af);
1735- found = af->findRec(&diskRec);
1736- CS_SET_DISK_4(diskRec.ar_repo_id_4, new_repo_id);
1737- CS_SET_DISK_8(diskRec.ar_offset_8, new_repo_offset);
1738-
1739- if (found)
1740- af->updateCurrentRec(&diskRec);
1741- else {
1742- CSException::logException(CS_CONTEXT, MS_ERR_NOT_FOUND, "Alias doesn't exists");
1743- af->addRec(&diskRec);
1744- }
1745-
1746- backtopool_(af);
1747-
1748- unlock_(this);
1749- exit_();
1750-}
1751-
1752-//------------------------
1753-// Check to see if the blob with the given repo_id
1754-// and repo_offset has the specified alias.
1755-bool MSAlias::hasBlobAlias(uint32_t repo_id, uint64_t repo_offset, const char *alias, bool *referenced)
1756-{
1757- bool found = false;
1758- MSRepoFile *repoFile;
1759- MSBlobHeadRec blob;
1760- uint8_t status;
1761- uint64_t offset;
1762- uint32_t alias_size = strlen(alias) +1;
1763- char blob_alias[BLOB_ALIAS_LENGTH +1];
1764-
1765- if (alias_size > BLOB_ALIAS_LENGTH)
1766- return false;
1767-
1768- enter_();
1769-
1770- repoFile = iDatabase_br->getRepoFileFromPool(repo_id, false);
1771- frompool_(repoFile);
1772-
1773- repoFile->read(&blob, repo_offset, sizeof(MSBlobHeadRec), sizeof(MSBlobHeadRec));
1774- status = CS_GET_DISK_1(blob.rb_status_1);
1775- if (IN_USE_BLOB_STATUS(status)) {
1776- offset = repo_offset + CS_GET_DISK_2(blob.rb_alias_offset_2);
1777-
1778- blob_alias[BLOB_ALIAS_LENGTH] = 0;
1779- if (repoFile->read(blob_alias, offset, alias_size, 0) == alias_size) {
1780- found = !my_strcasecmp(&my_charset_utf8_general_ci, blob_alias, alias);
1781- if (found)
1782- *referenced = (status == MS_BLOB_REFERENCED);
1783- }
1784- } else {
1785- CSException::logException(CS_CONTEXT, MS_ERR_ENGINE, "Deleted BLOB alias found. (Rebuild BLOB alias index.)");
1786- }
1787-
1788-
1789- backtopool_(repoFile);
1790-
1791- return_(found);
1792-}
1793-
1794-//------------------------
1795-bool MSAlias::findBlobByAlias( MSAliasFile *af, const char *alias, bool *referenced, uint32_t *repo_id, uint64_t *repo_offset)
1796-{
1797- bool found = false;
1798- uint32_t hash, l_repo_id, l_repo_offset;
1799- MSDiskAliasPtr diskRec;
1800- enter_();
1801-
1802- push_(af);
1803-
1804- hash = hashAlias(alias);
1805- diskRec = af->findRec(hash);
1806-
1807- while (diskRec && !found) {
1808- l_repo_id = CS_GET_DISK_4(diskRec->ar_repo_id_4);
1809- l_repo_offset = CS_GET_DISK_8(diskRec->ar_offset_8);
1810- if (hasBlobAlias(l_repo_id, l_repo_offset, alias, referenced))
1811- found = true;
1812- else
1813- diskRec = af->nextRec();
1814- }
1815-
1816- if (found) {
1817- if (repo_id)
1818- *repo_id = l_repo_id;
1819-
1820- if (repo_offset)
1821- *repo_offset = l_repo_offset;
1822- }
1823-
1824- release_(af);
1825- return_(found);
1826-}
1827-//------------------------
1828-bool MSAlias::findBlobByAlias( const char *alias, bool *referenced, uint32_t *repo_id, uint64_t *repo_offset)
1829-{
1830- bool found;
1831- enter_();
1832-
1833- MSAliasFile *af = getAliasFile();
1834- frompool_(af);
1835-
1836- found = findBlobByAlias(RETAIN(af), alias, referenced, repo_id, repo_offset);
1837-
1838- backtopool_(af);
1839- return_(found);
1840-}
1841-
1842-//------------------------
1843-bool MSAlias::blobAliasExists(uint32_t repo_id, uint64_t repo_offset, uint32_t alias_hash)
1844-{
1845- bool found;
1846- MSDiskAliasRec diskRec;
1847-
1848- CS_SET_DISK_4(diskRec.ar_repo_id_4, repo_id);
1849- CS_SET_DISK_8(diskRec.ar_offset_8, repo_offset);
1850- CS_SET_DISK_4(diskRec.ar_hash_4, alias_hash);
1851-
1852- enter_();
1853-
1854- MSAliasFile *af = getAliasFile();
1855- frompool_(af);
1856-
1857- found = af->findRec(&diskRec);
1858-
1859- backtopool_(af);
1860- return_(found);
1861-}
1862-
1863-/////////////////////////////////////
1864-MSSysMeta::MSSysMeta(MSAlias *msa)
1865-{
1866- md_myMSAlias = msa;
1867- md_isFileInUse = false;
1868- md_NextLink = md_PrevLink = NULL;
1869-
1870- mtab = MSMetaDataTable::newMSMetaDataTable(RETAIN(msa->iDatabase_br));
1871-}
1872-
1873-//------------------------
1874-MSSysMeta::~MSSysMeta()
1875-{
1876- if (mtab)
1877- mtab->release();
1878-
1879- if (md_myMSAlias)
1880- md_myMSAlias->release();
1881-}
1882-
1883-//------------------------
1884-void MSSysMeta::returnToPool()
1885-{
1886- enter_();
1887- push_(this);
1888-
1889-
1890- md_isFileInUse = false;
1891-
1892- if (!md_myMSAlias->iClosing) {
1893- lock_(&md_myMSAlias->iSysTablePoolLock); // It may be better if the pool had it's own lock.
1894- md_nextFile = md_myMSAlias->iSysTablePool;
1895- md_myMSAlias->iSysTablePool - this;
1896- unlock_(&md_myMSAlias->iSysTablePoolLock);
1897- }
1898-
1899- release_(this);
1900- exit_();
1901-}
1902-//------------------------
1903-bool MSSysMeta::matchAlias(uint32_t repo_id, uint64_t repo_offset, const char *alias)
1904-{
1905- mtab->seqScanInit();
1906- return mtab->matchAlias(repo_id, repo_offset, alias);
1907-}
1908-
1909-/////////////////////////////////////
1910-/////////////////////////////////////
1911-MSAliasFile::MSAliasFile(MSAliasFileShare *share)
1912-{
1913- ba_share = share;
1914- ba_isFileInUse = false;
1915- ba_NextLink = ba_PrevLink = NULL;
1916-
1917- iCurrentRec = 0;
1918- iBucketCache = NULL;
1919- iStartBucket = iCurrentBucket = NULL;
1920- iBucketChain = NULL;
1921- iLoading = false;
1922- ba_nextFile = NULL;
1923-
1924- iFile = CSFile::newFile(RETAIN(ba_share->msa_filePath));
1925- iFile->open(CSFile::DEFAULT);
1926-
1927-
1928-}
1929-
1930-//------------------------
1931-MSAliasFile::~MSAliasFile()
1932-{
1933- if (iFile)
1934- iFile->release();
1935-
1936- if (iBucketCache)
1937- cs_free(iBucketCache);
1938-}
1939-
1940-//------------------------
1941-void MSAliasFile::startLoad()
1942-{
1943- enter_();
1944-
1945- ASSERT(!iLoading);
1946-
1947-// iBucketCache = (MSADiskBucketRec*) cs_malloc(BUCKET_LIST_SIZE * sizeof(MSADiskBucketRec));
1948-// memset(iBucketCache, 0, BUCKET_LIST_SIZE * sizeof(MSADiskBucketRec));
1949- iLoading = true;
1950-
1951- exit_();
1952-}
1953-
1954-//------------------------
1955-void MSAliasFile::finishLoad()
1956-{
1957- enter_();
1958- ASSERT(iLoading);
1959- // Write the bucket cache to disk.
1960-// for (iCurrentBucket && iCurrentBucket->getSize()) {
1961- // To Be implemented.
1962-// }
1963-// cs_free(iBucketCache);
1964- iBucketCache = NULL;
1965- iLoading = false;
1966- exit_();
1967-}
1968-
1969-//------------------------
1970-void MSAliasFile::returnToPool()
1971-{
1972- enter_();
1973- push_(this);
1974-
1975- if (iLoading) {
1976- // If iLoading is still set then probably an exception has been thrown.
1977- try_(a) {
1978- finishLoad();
1979- }
1980- catch_(a)
1981- iLoading = false;
1982- cont_(a);
1983- }
1984-
1985- ba_isFileInUse = false;
1986-
1987- if (!ba_share->msa_closing) {
1988- lock_(&ba_share->msa_poolLock);
1989- ba_nextFile = ba_share->msa_pool;
1990- ba_share->msa_pool = this;
1991- unlock_(&ba_share->msa_poolLock);
1992- }
1993-
1994- release_(this);
1995- exit_();
1996-}
1997-
1998-//------------------------
1999-// The bucket chain is treated as a circular list.
2000-bool MSAliasFile::nextBucket(bool with_space)
2001-{
2002- bool have_bucket = false;
2003- enter_();
2004-
2005- while (!have_bucket){
2006- if (iCurrentBucket) {
2007- iCurrentBucket = iCurrentBucket->getNextLink();
2008- if (!iCurrentBucket)
2009- iCurrentBucket = iBucketChain->getFront();
2010- if (iCurrentBucket == iStartBucket)
2011- break;
2012- } else {
2013- iCurrentBucket = iBucketChain->getFront();
2014- iStartBucket = iCurrentBucket;
2015- }
2016-
2017- if ((iCurrentBucket->getSize() && !with_space) || (with_space && (iCurrentBucket->getSize() < NUM_RECORDS_PER_BUCKET))){
2018- // Only read the portion of the bucket containing records.
2019- iCurrentRec = iCurrentBucket->getEndOfRecords(); // The current record is set just beyond the last valid record.
2020- size_t size = iCurrentRec * sizeof(MSDiskAliasRec);
2021- iFile->read(iBucket, iCurrentBucket->bi_records_offset, size, size);
2022- have_bucket = true;
2023- }
2024- }
2025-
2026- return_(have_bucket);
2027-}
2028-
2029-//------------------------
2030-MSDiskAliasPtr MSAliasFile::nextRec()
2031-{
2032- MSDiskAliasPtr rec = NULL;
2033- bool have_rec;
2034- enter_();
2035-
2036- while ((!(have_rec = scanBucket())) && nextBucket(false));
2037-
2038- if (have_rec)
2039- rec = &(iBucket[iCurrentRec]);
2040-
2041- return_(rec);
2042-}
2043-
2044-//------------------------
2045-// When starting a search:
2046-// If a bucket is already loaded and it is in the correct bucket chain
2047-// then search it first. In this case then the search starts at the current
2048-// bucket in the chain.
2049-//
2050-// Searches are from back to front with the idea that the more recently
2051-// added objects will be seached for more often and they are more likely
2052-// to be at the end of the chain.
2053-MSDiskAliasPtr MSAliasFile::findRec(uint32_t hash)
2054-{
2055- MSDiskAliasPtr rec = NULL;
2056- MSABucketLinkedList *list = ba_share->getBucketChain(hash);
2057- enter_();
2058-
2059- CS_SET_DISK_4(iDiskHash_4, hash);
2060- if (list == iBucketChain) {
2061- // The search is performed back to front.
2062- iCurrentRec = iCurrentBucket->getEndOfRecords(); // Position the start just beyond the last valid record.
2063- iStartBucket = iCurrentBucket;
2064- if (scanBucket()) {
2065- rec = &(iBucket[iCurrentRec]);
2066- goto done;
2067- }
2068- } else {
2069- iBucketChain = list;
2070- iCurrentBucket = NULL;
2071- iStartBucket = NULL;
2072- }
2073-
2074- if (nextBucket(false))
2075- rec = nextRec();
2076-
2077-done:
2078- return_(rec);
2079-}
2080-
2081-//------------------------
2082-bool MSAliasFile::findRec(MSDiskAliasPtr theRec)
2083-{
2084- MSDiskAliasPtr aRec = NULL;
2085- bool found = false;
2086- enter_();
2087-
2088- aRec = findRec(CS_GET_DISK_4(theRec->ar_hash_4));
2089- while ( aRec && !found) {
2090- if (CS_EQ_DISK_4(aRec->ar_repo_id_4, theRec->ar_repo_id_4) && CS_EQ_DISK_8(aRec->ar_offset_8, theRec->ar_offset_8))
2091- found = true;
2092- else
2093- aRec = nextRec();
2094- }
2095- return_(found);
2096-}
2097-
2098-//------------------------
2099-void MSAliasFile::addRec(MSDiskAliasPtr new_rec)
2100-{
2101- MSABucketLinkedList *list = ba_share->getBucketChain(CS_GET_DISK_4(new_rec->ar_hash_4));
2102- enter_();
2103- lock_(&ba_share->msa_writeLock);
2104-
2105- if (iBucketChain != list) {
2106- iBucketChain = list;
2107- iCurrentBucket = NULL;
2108- iStartBucket = NULL;
2109- } else
2110- iStartBucket = iCurrentBucket;
2111-
2112- if ((iCurrentBucket && (iCurrentBucket->getSize() < NUM_RECORDS_PER_BUCKET)) || nextBucket(true)) { // Find a bucket with space in it for a record.
2113- uint32_t size = iCurrentBucket->getSize();
2114- uint32_t end_of_records = iCurrentBucket->getEndOfRecords();
2115-
2116- if (size == end_of_records) { // No holes in the recored list
2117- iCurrentRec = end_of_records;
2118- } else { // Search for the empty record
2119- iCurrentRec = end_of_records -2;
2120- while (iCurrentRec && !CS_IS_NULL_DISK_4(iBucket[iCurrentRec].ar_repo_id_4))
2121- iCurrentRec--;
2122-
2123- ASSERT(CS_IS_NULL_DISK_4(iBucket[iCurrentRec].ar_repo_id_4));
2124- }
2125-
2126- memcpy(&iBucket[iCurrentRec], new_rec, sizeof(MSDiskAliasRec)); // Add the record to the cached bucket.
2127-
2128- iCurrentBucket->recAdded(iFile, iCurrentRec); // update the current bucket header.
2129- } else { // A new bucket must be added to the chain.
2130- MSADiskBucketHeadRec new_bucket = {0};
2131- CSDiskValue8 disk_8_value;
2132- uint64_t new_bucket_offset;
2133- MSABucketInfo *next, *prev;
2134-
2135- next = iBucketChain->getFront();
2136- prev = iBucketChain->getBack();
2137-
2138- // Set the next and prev bucket offsets in the new bucket record.
2139- CS_SET_DISK_8(new_bucket.ab_prev_bucket_8, prev->bi_bucket_offset);
2140- CS_SET_DISK_8(new_bucket.ab_next_bucket_8, next->bi_bucket_offset);
2141-
2142- if (ba_share->msa_empty_buckets.getSize()) { // Get a bucket from the empty bucket list.
2143- MSABucketInfo *empty_bucket = ba_share->msa_empty_buckets.removeFront();
2144-
2145- new_bucket_offset = empty_bucket->bi_bucket_offset;
2146- empty_bucket->release();
2147-
2148- // Update the index file's empty bucket list
2149- if (ba_share->msa_empty_buckets.getSize() == 0)
2150- CS_SET_NULL_DISK_8(disk_8_value);
2151- else
2152- CS_SET_DISK_8(disk_8_value, iBucketChain->getFront()->bi_bucket_offset);
2153-
2154- iFile->write(&disk_8_value, offsetof(MSAliasHeadRec,ah_free_list_8) , 8);
2155- } else // There are no empty buckets so grow the file.
2156- new_bucket_offset = ba_share->msa_fileSize;
2157-
2158- // Write the new bucket's record header to the file
2159- iFile->write(&new_bucket, new_bucket_offset, sizeof(MSADiskBucketHeadRec));
2160-
2161- // Insert the new bucket into the bucket chain on the disk.
2162- CS_SET_DISK_8(disk_8_value, new_bucket_offset);
2163- iFile->write(&disk_8_value, prev->bi_bucket_offset + offsetof(MSADiskBucketHeadRec,ab_next_bucket_8), 8);
2164- iFile->write(&disk_8_value, next->bi_bucket_offset + offsetof(MSADiskBucketHeadRec,ab_prev_bucket_8), 8);
2165-
2166- // Update the file size in the file header if required
2167- if (ba_share->msa_fileSize == new_bucket_offset) {
2168- ba_share->msa_fileSize += sizeof(MSADiskBucketRec); // Note this is MSADiskBucketRec not MSADiskBucketHeadRec
2169-
2170- CS_SET_DISK_8(disk_8_value, ba_share->msa_fileSize);
2171- iFile->write(&disk_8_value, offsetof(MSAliasHeadRec,ah_file_size_8) , 8);
2172- }
2173-
2174- // Add the info rec into the bucket chain in RAM.
2175- iCurrentBucket = MSABucketInfo::newMSABucketInfo(new_bucket_offset, 1, 0);
2176- iBucketChain->addFront(iCurrentBucket);
2177- iCurrentRec = 0;
2178- }
2179-
2180- uint64_t offset;
2181- offset = iCurrentBucket->bi_records_offset + iCurrentRec * sizeof(MSDiskAliasRec);
2182-
2183- // Write the new index entry to the index file.
2184- iFile->write(new_rec, offset, sizeof(MSDiskAliasRec));
2185-
2186- unlock_(&ba_share->msa_writeLock);
2187-
2188- exit_();
2189-}
2190-//------------------------
2191-void MSAliasFile::deleteCurrentRec()
2192-{
2193- MSDiskAliasPtr rec = &(iBucket[iCurrentRec]);
2194- uint64_t offset;
2195- enter_();
2196-
2197- CS_SET_NULL_DISK_4(rec->ar_repo_id_4);
2198- offset = iCurrentBucket->bi_records_offset + iCurrentRec * sizeof(MSDiskAliasRec);
2199-
2200- lock_(&ba_share->msa_writeLock);
2201-
2202- // Update the index file. It is assumed that repo_id is the first 4 bytes of 'rec'.
2203- iFile->write(rec, offset, 4);
2204-
2205- iCurrentBucket->recRemoved(iFile, iCurrentRec, iBucket);
2206-
2207- unlock_(&ba_share->msa_writeLock);
2208-
2209- exit_();
2210-}
2211-
2212-//------------------------
2213-void MSAliasFile::updateCurrentRec(MSDiskAliasPtr update_rec)
2214-{
2215- uint64_t offset;
2216- enter_();
2217-
2218- // ASSERT that the updated rec still belongs to this bucket chain.
2219- ASSERT(ba_share->getBucketChain(CS_GET_DISK_4(update_rec->ar_hash_4)) == iBucketChain);
2220- ASSERT(!CS_IS_NULL_DISK_4(iBucket[iCurrentRec].ar_repo_id_4)); // We should not be updating a deleted record.
2221-
2222- lock_(&ba_share->msa_writeLock);
2223- offset = iCurrentBucket->bi_records_offset + iCurrentRec * sizeof(MSDiskAliasRec);
2224-
2225- // Update the record on disk.
2226- iFile->write(update_rec, offset, sizeof(MSDiskAliasRec));
2227-
2228- // Update the record in memory.
2229- CS_COPY_DISK_4(iBucket[iCurrentRec].ar_repo_id_4, update_rec->ar_repo_id_4);
2230- CS_COPY_DISK_8(iBucket[iCurrentRec].ar_offset_8, update_rec->ar_offset_8);
2231-
2232- unlock_(&ba_share->msa_writeLock);
2233- exit_();
2234-}
2235-
2236-
2237-//------------------------
2238-MSABucketInfo *MSABucketInfo::newMSABucketInfo(uint64_t offset, uint32_t num, uint32_t last)
2239-{
2240- MSABucketInfo *bucket;
2241- new_(bucket, MSABucketInfo(offset, num, last));
2242- return bucket;
2243-}
2244-//------------------------
2245-void MSABucketInfo::recRemoved(CSFile *iFile, uint32_t idx, MSDiskAliasRec bucket[])
2246-{
2247- MSADiskBucketHeadRec head;
2248- enter_();
2249-
2250- ASSERT(idx < bi_end_of_records);
2251-
2252- bi_num_recs--;
2253- if (!bi_num_recs) {
2254- // It would be nice to remove this bucket from the
2255- // bucket list and place it on the empty list.
2256- // Before this can be done a locking method would
2257- // be needed to block anyone from reading this
2258- // bucket while it was being moved.
2259- //
2260- // I haven't done this because I have been trying
2261- // to avoid read locks.
2262- bi_end_of_records = 0;
2263- } else if ((bi_end_of_records -1) == idx) {
2264- while (idx && CS_IS_NULL_DISK_4(bucket[idx].ar_repo_id_4))
2265- idx--;
2266-
2267- if ((idx ==0) && CS_IS_NULL_DISK_4(bucket[0].ar_repo_id_4))
2268- bi_end_of_records = 0;
2269- else
2270- bi_end_of_records = idx +1;
2271-
2272- ASSERT(bi_end_of_records >= bi_num_recs);
2273- }
2274-
2275- // Update the index file.
2276- CS_SET_DISK_4(head.ab_num_recs_4, bi_num_recs);
2277- CS_SET_DISK_4(head.ab_eor_rec_4, bi_end_of_records);
2278- iFile->write(&head.ab_num_recs_4, bi_bucket_offset + offsetof(MSADiskBucketHeadRec,ab_num_recs_4), 8);
2279- exit_();
2280-}
2281-
2282-//------------------------
2283-void MSABucketInfo::recAdded(CSFile *iFile, uint32_t idx)
2284-{
2285- MSADiskBucketHeadRec head;
2286- enter_();
2287-
2288- ASSERT(bi_num_recs < NUM_RECORDS_PER_BUCKET);
2289- ASSERT(idx < NUM_RECORDS_PER_BUCKET);
2290-
2291- bi_num_recs++;
2292- if (idx == bi_end_of_records)
2293- bi_end_of_records++;
2294-
2295- // Update the index file.
2296- CS_SET_DISK_4(head.ab_num_recs_4, bi_num_recs);
2297- CS_SET_DISK_4(head.ab_eor_rec_4, bi_end_of_records);
2298- iFile->write(&head.ab_num_recs_4, bi_bucket_offset + offsetof(MSADiskBucketHeadRec,ab_num_recs_4), 8);
2299- exit_();
2300-}
2301-
2302-//////////////////////////////////
2303-MSAliasFile *MSAliasFileShare::getPoolFile()
2304-{
2305- MSAliasFile *af;
2306- enter_();
2307-
2308- lock_(&msa_poolLock);
2309- if ((af = msa_pool)) {
2310- msa_pool = af->ba_nextFile;
2311- } else {
2312- new_(af, MSAliasFile(this));
2313- msa_poolFiles.addFront(af);
2314- }
2315- unlock_(&msa_poolLock);
2316-
2317- af->ba_nextFile = NULL;
2318- ASSERT(!af->ba_isFileInUse);
2319- af->ba_isFileInUse = true;
2320- af->retain();
2321-
2322- return_(af);
2323-}
2324-#endif // HAVE_ALIAS_SUPPORT
2325-
2326
2327=== removed file 'plugin/pbms/src/alias_ms.h'
2328--- plugin/pbms/src/alias_ms.h 2011-03-14 05:40:28 +0000
2329+++ plugin/pbms/src/alias_ms.h 1970-01-01 00:00:00 +0000
2330@@ -1,350 +0,0 @@
2331-/* Copyright (C) 2008 PrimeBase Technologies GmbH, Germany
2332- *
2333- * PrimeBase Media Stream for MySQL
2334- *
2335- * This program is free software; you can redistribute it and/or modify
2336- * it under the terms of the GNU General Public License as published by
2337- * the Free Software Foundation; either version 2 of the License, or
2338- * (at your option) any later version.
2339- *
2340- * This program is distributed in the hope that it will be useful,
2341- * but WITHOUT ANY WARRANTY; without even the implied warranty of
2342- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
2343- * GNU General Public License for more details.
2344- *
2345- * You should have received a copy of the GNU General Public License
2346- * along with this program; if not, write to the Free Software
2347- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
2348- *
2349- * Barry Leslie
2350- *
2351- * 2008-12-30
2352- *
2353- * H&G2JCtL
2354- *
2355- * BLOB alias index.
2356- *
2357- */
2358-
2359-
2360-#pragma once
2361-#ifndef __ALIAS_MS_H__
2362-#define __ALIAS_MS_H__
2363-#include <stddef.h>
2364-
2365-#include "defs_ms.h"
2366-#include "cslib/CSStorage.h"
2367-
2368-#define MS_ALIAS_FILE_MAGIC 0x5954228A
2369-#define MS_ALIAS_FILE_VERSION 1
2370-#define BLOB_ALIAS_LENGTH 1024
2371-#define INVALID_ALIAS_HASH ((uint32_t)-1)
2372-
2373-#ifdef HAVE_ALIAS_SUPPORT
2374-class MSOpenTable;
2375-class MSDatabase;
2376-class CSHTTPOutputStream;
2377-class MSMetaDataTable;
2378-
2379-#define ACTIVE_ALIAS_INDEX "ms_blob_alias.idx"
2380-
2381-#define NUM_RECORDS_PER_BUCKET 254 // 254 = bucket size of 4 K
2382-#define BUCKET_LIST_SIZE 1024
2383-
2384-typedef struct MSAliasHead {
2385- CSDiskValue4 ah_magic_4; /* Table magic number. */
2386- CSDiskValue2 ah_version_2; /* The header version. */
2387- CSDiskValue2 ah_head_size_2; /* The size of the header. */
2388- CSDiskValue8 ah_file_size_8; /* The size of the file. */
2389-
2390- CSDiskValue8 ah_free_list_8; /* The offset of the first bucket in the free list. */
2391-
2392- CSDiskValue2 ah_num_buckets_2; /* The number of bucket chains in the index. (BUCKET_LIST_SIZE when created)*/
2393- CSDiskValue4 ah_bucket_size_4; /* The size of each bucket. (NUM_RECORDS_PER_BUCKET when created)*/
2394-
2395-} MSAliasHeadRec, *MSAliasHeadPtr;
2396-
2397-/*
2398- * When a record is freed ba_repo_id_4 is set to zero
2399-*/
2400-typedef struct MSDiskAliasRec {
2401- CSDiskValue4 ar_repo_id_4; /* File ID. Not zero when allocated. */
2402- CSDiskValue8 ar_offset_8; /* Offset into the file of the BLOB. */
2403- CSDiskValue4 ar_hash_4; /* The hash value of the alias string. (Is assumed to be at the end of the structure.*/
2404-} MSDiskAliasRec, *MSDiskAliasPtr;
2405-
2406-typedef struct MSADiskBucketHead {
2407- CSDiskValue8 ab_prev_bucket_8; /* The file offset of the previouse bucket in the chain. */
2408- CSDiskValue8 ab_next_bucket_8; /* The file offset of the next bucket in the chain. */
2409- CSDiskValue4 ab_num_recs_4; /* The number of used record in the bucket. */
2410- CSDiskValue4 ab_eor_rec_4; /* (End Of Records) The position of the first free record after all the records in the bucket. */
2411-} MSADiskBucketHeadRec, *MSADiskBucketHeadPtr;
2412-
2413-typedef struct MSADiskBucket {
2414- MSADiskBucketHeadRec ab_heaher;
2415- MSDiskAliasRec ab_records[NUM_RECORDS_PER_BUCKET]; /* The start of the records in the bucket. */
2416-} MSADiskBucketRec, *MSADiskBucketPtr;
2417-
2418-/*
2419- * MSABucketInfo stores bucket information in RAM.
2420- */
2421-class MSABucketInfo: public CSOrderKey {
2422-public:
2423-
2424- MSABucketInfo(uint64_t offset, uint32_t num, uint32_t end_of_records):
2425- bi_bucket_offset(offset),
2426- bi_records_offset(offset + offsetof(MSADiskBucketRec, ab_records)),
2427- bi_num_recs(num),
2428- bi_end_of_records(end_of_records),
2429- bi_NextLink(NULL),
2430- bi_PrevLink(NULL)
2431- {}
2432-
2433- uint64_t bi_bucket_offset; /* The file offset of the bucket. */
2434-
2435- uint64_t bi_records_offset; /* The file offset of the first record in the bucket. */
2436-
2437- // Required method for item in a CSLinkedList.
2438- virtual MSABucketInfo *getNextLink() { return bi_NextLink; }
2439- virtual MSABucketInfo *getPrevLink() { return bi_PrevLink; }
2440- virtual void setNextLink(CSObject *link) { bi_NextLink = (MSABucketInfo*)link; }
2441- virtual void setPrevLink(CSObject *link) { bi_PrevLink = (MSABucketInfo*)link; }
2442-
2443- virtual CSObject *getKey() { return this;}
2444- virtual int compareKey(CSOrderKey *x) {
2445- MSABucketInfo *key = (MSABucketInfo *) x;
2446-
2447- if (bi_bucket_offset != key->bi_bucket_offset)
2448- return 0;
2449-
2450- return (bi_bucket_offset < key->bi_bucket_offset)? -1: 1;
2451- }
2452-
2453- static MSABucketInfo *newMSABucketInfo(uint64_t offset, uint32_t num = 0, uint32_t last = 0);
2454-
2455- uint32_t getSize() { return bi_num_recs;}
2456- uint32_t getEndOfRecords() { return bi_end_of_records;}
2457- void recAdded(CSFile *iFile, uint32_t idx);
2458- void recRemoved(CSFile *iFile, uint32_t idx, MSDiskAliasRec bucket[]);
2459-
2460-private:
2461- // (bi_end_of_records -1) is the index of the last valid record in the bucket.
2462- // free records can actually appear any where in the bucket unless it has
2463- // just been compressed.
2464- uint32_t bi_num_recs; /* The number of records in the bucket. */
2465- uint32_t bi_end_of_records; /* The index of the start of the free records in the bucket. */
2466-
2467- MSABucketInfo *bi_NextLink;
2468- MSABucketInfo *bi_PrevLink;
2469-};
2470-
2471-class MSABucketLinkedList: public CSLinkedList {
2472-public:
2473-
2474- /* Value is returned referenced. */
2475- MSABucketInfo *removeBack() { return (MSABucketInfo*) CSLinkedList::removeBack();}
2476-
2477- /* Value is returned NOT referenced. */
2478- MSABucketInfo *getBack(){ return (MSABucketInfo*) CSLinkedList::getBack();}
2479-
2480- /* Value is returned NOT referenced. */
2481- MSABucketInfo *getFront(){ return (MSABucketInfo*) CSLinkedList::getFront();}
2482-
2483- /* Value is returned referenced. */
2484- MSABucketInfo *removeFront(){ return (MSABucketInfo*) CSLinkedList::removeFront();}
2485-};
2486-
2487-typedef struct MSAliasRec {
2488- uint32_t repo_id;
2489- uint64_t repo_offset;
2490- uint32_t alias_hash;
2491-} MSAliasRec, *MSAliasPtr;
2492-
2493-class MSAliasFile : public CSPooled, public CSRefObject {
2494-public:
2495- class MSAliasFileShare *ba_share;
2496- bool ba_isFileInUse;
2497- MSAliasFile *ba_nextFile; /* Next file available in the pool */
2498-
2499- MSAliasFile(MSAliasFileShare *share);
2500- virtual ~MSAliasFile();
2501-
2502- // Required method for CSPool item.
2503- virtual void returnToPool();
2504-
2505-
2506- // Required method for item in a CSLinkedList.
2507- virtual CSObject *getNextLink() { return ba_NextLink; }
2508- virtual CSObject *getPrevLink() { return ba_PrevLink; }
2509- virtual void setNextLink(CSObject *link) { ba_NextLink = link; }
2510- virtual void setPrevLink(CSObject *link) { ba_PrevLink = link; }
2511-
2512- // Index file operations.
2513- MSDiskAliasPtr findRec(uint32_t hash);
2514- MSDiskAliasPtr nextRec();
2515- void addRec(MSDiskAliasPtr rec);
2516- void deleteCurrentRec();
2517- void updateCurrentRec(MSDiskAliasPtr rec);
2518- bool findRec(MSDiskAliasPtr rec);
2519-
2520- /* When a load is inprogress locks are not required and writes are batched. */
2521- void startLoad();
2522- void finishLoad();
2523-
2524-
2525-private:
2526- bool nextBucket(bool with_space);
2527-
2528- bool scanBucket()
2529- {
2530- while (iCurrentRec) {
2531- iCurrentRec--;
2532- if ( CS_EQ_DISK_4(iDiskHash_4, iBucket[iCurrentRec].ar_hash_4)
2533- && !CS_IS_NULL_DISK_4(iBucket[iCurrentRec].ar_repo_id_4))
2534- return true;
2535- }
2536- return false;
2537- }
2538-
2539- CSFile *iFile; // The index file.
2540-
2541- bool iLoading;
2542- MSADiskBucketRec *iBucketCache; // The bucket cache is used during index loading in single thread mode.
2543- MSDiskAliasRec iBucket[NUM_RECORDS_PER_BUCKET];// The current bucket loaded from disk.
2544- MSABucketLinkedList *iBucketChain; // The bucket list for the current hash value.
2545- MSABucketInfo *iStartBucket; // The file offset of the bucket the search started at.
2546- MSABucketInfo *iCurrentBucket;// The currnet bucket, NULL if no bucket is loaded.
2547-
2548- CSDiskValue4 iDiskHash_4; // The current hash value we are looking for in disk byte order.
2549- uint32_t iCurrentRec; // The current record position in the current bucket.
2550-
2551- CSObject *ba_NextLink;
2552- CSObject *ba_PrevLink;
2553-
2554-};
2555-
2556-//===========================================
2557-class MSAliasFileShare: public CSObject {
2558-public:
2559- MSAliasFileShare(CSPath *path):
2560- msa_filePath(path),
2561- msa_fileSize(0),
2562- msa_pool(NULL),
2563- msa_closing(false)
2564- {
2565- bool isdir = false;
2566- if (path->exists(&isdir))
2567- msa_fileSize = path->getSize();
2568- }
2569-
2570- ~MSAliasFileShare()
2571- {
2572- msa_poolFiles.clear();
2573- if (msa_filePath)
2574- msa_filePath->release();
2575-
2576- for (uint32_t i =0; i < BUCKET_LIST_SIZE; i++)
2577- msa_buckets[i].clear();
2578-
2579- msa_empty_buckets.clear();
2580- }
2581-
2582- void close() { msa_poolFiles.clear();}
2583-
2584- MSABucketLinkedList *getBucketChain(uint32_t hash) { return msa_buckets + (hash % BUCKET_LIST_SIZE); }
2585- MSAliasFile *getPoolFile();
2586-
2587- CSLinkedList msa_poolFiles; /* A list of all files in this pool */
2588- uint64_t msa_fileSize;
2589- CSPath *msa_filePath;
2590- CSLock msa_writeLock;
2591- MSAliasFile *msa_pool; /* A list of files currently not in use. */
2592-
2593- CSLock msa_poolLock;
2594- bool msa_closing;
2595- MSABucketLinkedList msa_empty_buckets; /* A list of unused buckets. */
2596-
2597- MSABucketLinkedList msa_buckets[BUCKET_LIST_SIZE]; /* An array of bucket chains. */
2598-};
2599-
2600-//===========================================
2601-class MSSysMeta : public CSRefObject, public CSPooled {
2602-public:
2603- class MSAlias *md_myMSAlias;
2604- bool md_isFileInUse;
2605- MSSysMeta *md_nextFile; /* Next file available in the pool */
2606-
2607- MSSysMeta(MSAlias *msa);
2608- virtual ~MSSysMeta();
2609-
2610- virtual void returnToPool();
2611-
2612- virtual CSObject *getNextLink() { return md_NextLink; }
2613- virtual CSObject *getPrevLink() { return md_PrevLink; }
2614- virtual void setNextLink(CSObject *link) { md_NextLink = link; }
2615- virtual void setPrevLink(CSObject *link) { md_PrevLink = link; }
2616-
2617- bool matchAlias(uint32_t repo_id, uint64_t repo_offset, const char *alias);
2618-
2619-private:
2620- MSMetaDataTable *mtab;
2621- CSObject *md_NextLink;
2622- CSObject *md_PrevLink;
2623-
2624-};
2625-
2626-//===========================================
2627-class MSAlias : public CSSharedRefObject {
2628-public:
2629-
2630- MSAlias(MSDatabase *db_noref);
2631- ~MSAlias();
2632-
2633- void ma_open(const char *file_name = ACTIVE_ALIAS_INDEX);
2634- void ma_close();
2635- void ma_delete() {iDelete = true;}
2636-
2637- uint32_t addAlias(uint32_t repo_id, uint64_t repo_offset, const char *alias);
2638-private:
2639- void addAlias(MSAliasFile *af, MSAliasRec *rec);
2640-public:
2641- void deleteAlias(MSDiskAliasPtr diskRec);
2642- void deleteAlias(uint32_t repo_id, uint64_t repo_offset, uint32_t alias_hash);
2643- void resetAlias(uint32_t old_repo_id, uint64_t old_repo_offset, uint32_t alias_hash, uint32_t new_repo_id, uint64_t new_repo_offset);
2644-
2645- bool findBlobByAlias(const char *alias, bool *referenced, uint32_t *repo_id = NULL, uint64_t *repo_offset = NULL);
2646- bool blobAliasExists(uint32_t repo_id, uint64_t repo_offset, uint32_t alias_hash);
2647-private:
2648- bool findBlobByAlias(MSAliasFile *af, const char *alias, bool *referenced, uint32_t *repo_id = NULL, uint64_t *repo_offset = NULL);
2649-public:
2650- static uint32_t hashAlias(const char *ptr);
2651-
2652- void MSAliasBuild();
2653-
2654- friend class MSAliasFile;
2655- friend class MSSysMeta;
2656-
2657-private:
2658- MSDatabase *iDatabase_br; // This is a back reference so this reference is not counted.
2659- CSPath *iFilePath;
2660-
2661- bool iClosing;
2662- bool iDelete; // true when the alias index file should be deleted.
2663-
2664- MSAliasFileShare *iFileShare; // File information shared between all files in the pool.
2665-
2666- CSLock iSysTablePoolLock;
2667- MSSysMeta *iSysTablePool; /* A list of files currently not in use. */
2668- CSLinkedList iPoolSysTables; /* A list of all files in this pool */
2669-
2670- MSAliasFile *getAliasFile() { return iFileShare->getPoolFile();}
2671- void buildAliasIndex();
2672- void MSAliasCompress(CSFile *fa, CSSortedList *freeList, MSABucketLinkedList *bucketChain);
2673- void MSAliasLoad();
2674- bool hasBlobAlias(uint32_t repo_id, uint64_t repo_offset, const char *alias, bool *referenced);
2675-};
2676-
2677-
2678-#endif //HAVE_ALIAS_SUPPORT
2679-
2680-#endif // __ALIAS_MS_H__
2681
2682=== removed file 'plugin/pbms/src/api_ms.cc'
2683--- plugin/pbms/src/api_ms.cc 2010-12-18 04:43:40 +0000
2684+++ plugin/pbms/src/api_ms.cc 1970-01-01 00:00:00 +0000
2685@@ -1,270 +0,0 @@
2686-#ifdef NOT_USED_IN_ANY_THING
2687-
2688-/* Copyright (C) 2008 PrimeBase Technologies GmbH, Germany
2689- *
2690- * PrimeBase Media Stream for MySQL
2691- *
2692- * This program is free software; you can redistribute it and/or modify
2693- * it under the terms of the GNU General Public License as published by
2694- * the Free Software Foundation; either version 2 of the License, or
2695- * (at your option) any later version.
2696- *
2697- * This program is distributed in the hope that it will be useful,
2698- * but WITHOUT ANY WARRANTY; without even the implied warranty of
2699- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
2700- * GNU General Public License for more details.
2701- *
2702- * You should have received a copy of the GNU General Public License
2703- * along with this program; if not, write to the Free Software
2704- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
2705- *
2706- * Barry Leslie
2707- *
2708- * 2007-11-25
2709- *
2710- * H&G2JCtL
2711- *
2712- */
2713-
2714-#include "cslib/CSConfig.h"
2715-#include "cslib/CSGlobal.h"
2716-#include "cslib/CSLog.h"
2717-#include "cslib/CSStrUtil.h"
2718-#include "cslib/CSHTTPStream.h"
2719-#include "cslib/CSStream.h"
2720-
2721-#include "repository_ms.h"
2722-#include "open_table_ms.h"
2723-#include "mysql_ms.h"
2724-
2725-//-----------------------------------------------------------------------------------------------
2726-void PBMSGetError(void *v_bs_thread, PBMSResultPtr result)
2727-{
2728- CSThread *ms_thread = (CSThread*)v_bs_thread;
2729-
2730- ASSERT(ms_thread);
2731- memset(result, 0, sizeof(PBMSResultRec));
2732-
2733- result->mr_code = ms_thread->myException.getErrorCode();
2734- cs_strcpy(MS_RESULT_MESSAGE_SIZE, result->mr_message, ms_thread->myException.getMessage());
2735-}
2736-
2737-//-----------------------------------------------------------------------------------------------
2738-void *PBMSInitBlobStreamingThread(char *thread_name, PBMSResultPtr result)
2739-{
2740- CSThread *ms_thread = new CSThread( NULL);
2741-
2742- if (!ms_thread) {
2743- memset(result, 0, sizeof(PBMSResultRec));
2744- result->mr_code = ENOMEM;
2745- cs_strcpy(MS_RESULT_MESSAGE_SIZE, result->mr_message, "CSThread::newThread() failed.");
2746- return NULL;
2747- }
2748-
2749- ms_thread->pbms_api_owner = true;
2750- if (!CSThread::attach(ms_thread)) {
2751- memset(result, 0, sizeof(PBMSResultRec));
2752- result->mr_code = ms_thread->myException.getErrorCode();
2753- cs_strcpy(MS_RESULT_MESSAGE_SIZE, result->mr_message, ms_thread->myException.getMessage());
2754- ms_thread->release();
2755- ms_thread = NULL;
2756- } else
2757- ms_thread->threadName = CSString::newString(thread_name);
2758-
2759- return ms_thread;
2760-}
2761-
2762-
2763-//-----------------------------------------------------------------------------------------------
2764-void PBMSDeinitBlobStreamingThread(void *v_bs_thread)
2765-{
2766- CSThread *ms_thread = (CSThread*)v_bs_thread;
2767-
2768- ASSERT(ms_thread);
2769-
2770- CSThread::detach(ms_thread);
2771- // ms_thread->release(); Don't do this. Ownership of the thread is passed to the attach call so the thread is released when it is detached.
2772-}
2773-
2774-//-----------------------------------------------------------------------------------------------
2775-bool PBMSCreateBlob(PBMSBlobIDPtr blob_id, char *database_name, uint64_t size)
2776-{
2777- MSOpenTable *otab = NULL;
2778- CSString *iTableURI = NULL;
2779- CSString *CSContenttype = NULL;
2780- bool done_ok = true;
2781-
2782- enter_();
2783-
2784- try_(a) {
2785- otab = MSTableList::getOpenTableForDB(MSDatabase::getDatabaseID(database_name, false));
2786-
2787- otab->createBlob(blob_id, size, NULL, 0);
2788- }
2789-
2790- catch_(a) {
2791- done_ok = false;
2792- }
2793- cont_(a);
2794-
2795- exit:
2796- if (otab)
2797- otab->returnToPool();
2798-
2799- if (CSContenttype)
2800- CSContenttype->release();
2801-
2802- if (iTableURI)
2803- iTableURI->release();
2804-
2805- return_(done_ok);
2806-}
2807-
2808-//-----------------------------------------------------------------------------------------------
2809-bool PBMSWriteBlob(PBMSBlobIDPtr blob_id, char *data, size_t size, size_t offset)
2810-{
2811- MSOpenTable *otab;
2812- MSRepoFile *repo_file;
2813- bool done_ok = true;
2814-
2815- enter_();
2816-
2817- try_(a) {
2818- if (!(otab = MSTableList::getOpenTableForDB(blob_id->bi_db_id))) {
2819- char buffer[CS_EXC_MESSAGE_SIZE];
2820- char id_str[12];
2821-
2822- snprintf(id_str, 12, "%"PRIu32"", blob_id->bi_db_id);
2823-
2824- cs_strcpy(CS_EXC_MESSAGE_SIZE, buffer, "Unknown database id # ");
2825- cs_strcat(CS_EXC_MESSAGE_SIZE, buffer, id_str);
2826- CSException::throwException(CS_CONTEXT, MS_ERR_UNKNOWN_DB, buffer);
2827- }
2828- frompool_(otab);
2829- repo_file = otab->getDB()->getRepoFileFromPool( blob_id->bi_tab_id, false);
2830- frompool_(repo_file);
2831- // It is assumed that at this point the blob is a repository blob and so the
2832- // blob_id->bi_blob_id is actually the repository blob offset.
2833- repo_file->writeBlobChunk(blob_id, blob_id->bi_blob_id, offset, size, data);
2834- backtopool_(repo_file);
2835- backtopool_(otab);
2836-
2837- }
2838- catch_(a) {
2839- done_ok = false;
2840- }
2841-
2842- cont_(a);
2843-
2844- return_(done_ok);
2845-}
2846-
2847-//-----------------------------------------------------------------------------------------------
2848-bool PBMSReadBlob(PBMSBlobIDPtr blob_id, char *buffer, size_t *size, size_t offset)
2849-{
2850- MSOpenTable *otab;
2851- MSRepoFile *repo_file;
2852- bool done_ok = true, is_repository_blob;
2853-
2854- enter_();
2855-
2856- is_repository_blob = (blob_id->bi_blob_type == MS_URL_TYPE_REPO);
2857- try_(a) {
2858- if (!(otab = MSTableList::getOpenTableByID(blob_id->bi_db_id, blob_id->bi_tab_id))) {
2859- char buffer[CS_EXC_MESSAGE_SIZE];
2860- char id_str[12];
2861-
2862-
2863- cs_strcpy(CS_EXC_MESSAGE_SIZE, buffer, "Unknown database: ID # ");
2864- snprintf(id_str, 12, "%"PRIu32"", blob_id->bi_db_id);
2865- cs_strcat(CS_EXC_MESSAGE_SIZE, buffer, id_str);
2866- cs_strcpy(CS_EXC_MESSAGE_SIZE, buffer, " or table: ID #");
2867- snprintf(id_str, 12, "%"PRIu32"", blob_id->bi_tab_id);
2868- cs_strcat(CS_EXC_MESSAGE_SIZE, buffer, id_str);
2869- CSException::throwException(CS_CONTEXT, MS_ERR_UNKNOWN_DB, buffer);
2870- }
2871- uint32_t repo_id;
2872- uint64_t rep_offset;
2873-
2874-
2875- frompool_(otab);
2876- if (is_repository_blob) {
2877- repo_id = blob_id->bi_tab_id;
2878- rep_offset = blob_id->bi_blob_id;
2879- } else {
2880- uint64_t blob_size;
2881- uint16_t header_size;
2882- otab->getDBTable()->readBlobHandle(otab, blob_id->bi_blob_id, &(blob_id->bi_auth_code), &repo_id, &rep_offset, &blob_size, &header_size, true);
2883- }
2884-
2885- repo_file = otab->getDB()->getRepoFileFromPool( repo_id, false);
2886- frompool_(repo_file);
2887- *size = repo_file->readBlobChunk(blob_id, rep_offset, offset, *size, buffer);
2888- backtopool_(repo_file);
2889- backtopool_(otab);
2890-
2891- }
2892- catch_(a) {
2893- done_ok = false;
2894- }
2895-
2896- cont_(a);
2897-
2898- return_(done_ok);
2899-}
2900-
2901-//-----------------------------------------------------------------------------------------------
2902-bool PBMSIDToURL(PBMSBlobIDPtr blob_id, PBMSBlobURLPtr url)
2903-{
2904- MSBlobURL ms_blob;
2905-
2906- ms_blob.bu_db_id = blob_id->bi_db_id;
2907- ms_blob.bu_blob_id = blob_id->bi_blob_id;
2908- ms_blob.bu_blob_ref_id = blob_id->bi_blob_ref_id;
2909- ms_blob.bu_tab_id = blob_id->bi_tab_id;
2910- ms_blob.bu_auth_code = blob_id->bi_auth_code;
2911- ms_blob.bu_type = blob_id->bi_blob_type;
2912- ms_blob.bu_blob_size = blob_id->bi_blob_size;
2913- ms_blob.bu_server_id = ms_my_get_server_id();
2914-
2915- PBMSBlobURLTools::buildBlobURL(&ms_blob, url);
2916- return true;
2917-}
2918-
2919-//-----------------------------------------------------------------------------------------------
2920-bool PBMSURLToID(char *url, PBMSBlobIDPtr blob_id)
2921-{
2922- MSBlobURL ms_blob;
2923- bool done_ok = true;
2924- enter_();
2925-
2926- try_(a) {
2927-
2928- if (!PBMSBlobURLTools::couldBeURL(url, &ms_blob)){
2929- char buffer[CS_EXC_MESSAGE_SIZE];
2930-
2931- cs_strcpy(CS_EXC_MESSAGE_SIZE, buffer, "Incorrect URL: ");
2932- cs_strcat(CS_EXC_MESSAGE_SIZE, buffer, url);
2933- CSException::throwException(CS_CONTEXT, MS_ERR_INCORRECT_URL, buffer);
2934- }
2935-
2936- blob_id->bi_db_id = ms_blob.bu_db_id;
2937- blob_id->bi_blob_id = ms_blob.bu_blob_id;
2938- blob_id->bi_blob_ref_id = ms_blob.bu_blob_ref_id;
2939- blob_id->bi_tab_id = ms_blob.bu_tab_id;
2940- blob_id->bi_auth_code = ms_blob.bu_auth_code;
2941- blob_id->bi_blob_type = ms_blob.bu_type;
2942- blob_id->bi_blob_size = ms_blob.bu_blob_size;
2943-
2944- }
2945- catch_(a) {
2946- done_ok = false;
2947- }
2948-
2949- cont_(a);
2950-
2951- return_(done_ok);
2952-}
2953-
2954-
2955-#endif // NOT_USED_IN_ANY_THING
2956
2957=== removed file 'plugin/pbms/src/backup_ms.cc'
2958--- plugin/pbms/src/backup_ms.cc 2011-04-20 22:18:30 +0000
2959+++ plugin/pbms/src/backup_ms.cc 1970-01-01 00:00:00 +0000
2960@@ -1,733 +0,0 @@
2961-/* Copyright (C) 2009 PrimeBase Technologies GmbH, Germany
2962- *
2963- * PrimeBase Media Stream for MySQL
2964- *
2965- * This program is free software; you can redistribute it and/or modify
2966- * it under the terms of the GNU General Public License as published by
2967- * the Free Software Foundation; either version 2 of the License, or
2968- * (at your option) any later version.
2969- *
2970- * This program is distributed in the hope that it will be useful,
2971- * but WITHOUT ANY WARRANTY; without even the implied warranty of
2972- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
2973- * GNU General Public License for more details.
2974- *
2975- * You should have received a copy of the GNU General Public License
2976- * along with this program; if not, write to the Free Software
2977- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
2978- *
2979- * Barry Leslie
2980- *
2981- * 2009-05-29
2982- *
2983- * H&G2JCtL
2984- *
2985- * Repository backup.
2986- *
2987- * The backup is done by creating a new database with the same name and ID in the
2988- * backup location. Then the pbms_dump table in the source database is initialized
2989- * for a sequential scan for backup. This has the effect of locking all current repository
2990- * files. Then the equvalent of 'insert into dst_db.pbms_dump (select * from src_db.pbms_dump);'
2991- * is performed.
2992- *
2993- */
2994-
2995-#ifdef DRIZZLED
2996-#include <config.h>
2997-#include <drizzled/common.h>
2998-#include <drizzled/session.h>
2999-#include <drizzled/table.h>
3000-#include <drizzled/message/table.pb.h>
3001-#include <drizzled/charset.h>
3002-#include <drizzled/table_proto.h>
3003-#include <drizzled/field.h>
3004-#include <drizzled/field/varstring.h>
3005-#endif
3006-
3007-#include "cslib/CSConfig.h"
3008-
3009-#include <sys/types.h>
3010-#include <inttypes.h>
3011-
3012-#include "cslib/CSGlobal.h"
3013-#include "cslib/CSStrUtil.h"
3014-#include "cslib/CSStorage.h"
3015-
3016-#include "defs_ms.h"
3017-#include "system_table_ms.h"
3018-#include "open_table_ms.h"
3019-#include "table_ms.h"
3020-#include "database_ms.h"
3021-#include "repository_ms.h"
3022-#include "backup_ms.h"
3023-#include "transaction_ms.h"
3024-#include "systab_variable_ms.h"
3025-#include "systab_backup_ms.h"
3026-
3027-uint32_t MSBackupInfo::gMaxInfoRef;
3028-CSSyncSparseArray *MSBackupInfo::gBackupInfo;
3029-
3030-//==========================================
3031-MSBackupInfo::MSBackupInfo( uint32_t id,
3032- const char *name,
3033- uint32_t db_id_arg,
3034- time_t start,
3035- time_t end,
3036- bool _isDump,
3037- const char *location,
3038- uint32_t cloudRef_arg,
3039- uint32_t cloudBackupNo_arg ):
3040- backupRefId(id),
3041- db_name(NULL),
3042- db_id(db_id_arg),
3043- startTime(start),
3044- completionTime(end),
3045- dump(_isDump),
3046- isRunning(false),
3047- backupLocation(NULL),
3048- cloudRef(cloudRef_arg),
3049- cloudBackupNo(cloudBackupNo_arg)
3050-{
3051- db_name = CSString::newString(name);
3052- if (location && *location)
3053- backupLocation = CSString::newString(location);
3054-}
3055-
3056-//-------------------------------
3057-MSBackupInfo::~MSBackupInfo()
3058-{
3059- if (db_name)
3060- db_name->release();
3061-
3062- if (backupLocation)
3063- backupLocation->release();
3064-}
3065-
3066-//-------------------------------
3067-void MSBackupInfo::startBackup(MSDatabase *pbms_db)
3068-{
3069- MSDatabase *src_db;
3070-
3071- enter_();
3072- push_(pbms_db);
3073-
3074- src_db = MSDatabase::getDatabase(db_id);
3075- push_(src_db);
3076-
3077- startTime = time(NULL);
3078-
3079- src_db->startBackup(RETAIN(this));
3080- release_(src_db);
3081-
3082- isRunning = true;
3083-
3084- pop_(pbms_db);
3085- MSBackupTable::saveTable(pbms_db);
3086- exit_();
3087-}
3088-
3089-//-------------------------------
3090-class StartDumpCleanUp : public CSRefObject {
3091- bool do_cleanup;
3092- uint32_t ref_id;
3093-
3094- public:
3095-
3096- StartDumpCleanUp(): CSRefObject(),
3097- do_cleanup(false){}
3098-
3099- ~StartDumpCleanUp()
3100- {
3101- if (do_cleanup) {
3102- MSBackupInfo::gBackupInfo->remove(ref_id);
3103- }
3104- }
3105-
3106- void setCleanUp(uint32_t id)
3107- {
3108- ref_id = id;
3109- do_cleanup = true;
3110- }
3111-
3112- void cancelCleanUp()
3113- {
3114- do_cleanup = false;
3115- }
3116-
3117-};
3118-
3119-MSBackupInfo *MSBackupInfo::startDump(MSDatabase *db, uint32_t cloud_ref, uint32_t backup_no)
3120-{
3121- MSBackupInfo *info;
3122- uint32_t ref_id;
3123- StartDumpCleanUp *cleanup;
3124-
3125- enter_();
3126- push_(db);
3127- lock_(gBackupInfo);
3128-
3129- ref_id = gMaxInfoRef++;
3130- new_(info, MSBackupInfo(ref_id, db->myDatabaseName->getCString(), db->myDatabaseID, time(NULL), 0, true, NULL, cloud_ref, backup_no));
3131- push_(info);
3132-
3133- gBackupInfo->set(ref_id, RETAIN(info));
3134-
3135- info->isRunning = true;
3136-
3137- pop_(info);
3138- unlock_(gBackupInfo);
3139- push_(info);
3140-
3141- // Create a cleanup object to handle cleanup
3142- // after a possible exception.
3143- new_(cleanup, StartDumpCleanUp());
3144- push_(cleanup);
3145- cleanup->setCleanUp(ref_id);
3146-
3147- MSBackupTable::saveTable(RETAIN(db));
3148-
3149- cleanup->cancelCleanUp();
3150- release_(cleanup);
3151-
3152- pop_(info);
3153- release_(db);
3154-
3155- return_(info);
3156-}
3157-//-------------------------------
3158-void MSBackupInfo::backupCompleted(MSDatabase *db)
3159-{
3160- completionTime = time(NULL);
3161- isRunning = false;
3162- MSBackupTable::saveTable(db);
3163-}
3164-
3165-//-------------------------------
3166-void MSBackupInfo::backupTerminated(MSDatabase *db)
3167-{
3168- enter_();
3169- push_(db);
3170- lock_(gBackupInfo);
3171-
3172- gBackupInfo->remove(backupRefId);
3173- unlock_(gBackupInfo);
3174-
3175- pop_(db);
3176- MSBackupTable::saveTable(db);
3177- exit_();
3178-}
3179-
3180-//==========================================
3181-MSBackup::MSBackup():
3182-CSDaemon(NULL),
3183-bu_info(NULL),
3184-bu_BackupList(NULL),
3185-bu_Compactor(NULL),
3186-bu_BackupRunning(false),
3187-bu_State(BU_COMPLETED),
3188-bu_SourceDatabase(NULL),
3189-bu_Database(NULL),
3190-bu_dst_dump(NULL),
3191-bu_src_dump(NULL),
3192-bu_size(0),
3193-bu_completed(0),
3194-bu_ID(0),
3195-bu_start_time(0),
3196-bu_TransactionManagerSuspended(false)
3197-{
3198-}
3199-
3200-MSBackup *MSBackup::newMSBackup(MSBackupInfo *info)
3201-{
3202- MSBackup *bu;
3203- enter_();
3204-
3205- push_(info);
3206-
3207- new_(bu, MSBackup());
3208- push_(bu);
3209- bu->bu_Database = MSDatabase::getBackupDatabase(RETAIN(info->backupLocation), RETAIN(info->db_name), info->db_id, true);
3210- pop_(bu);
3211-
3212- bu->bu_info = info;
3213- pop_(info);
3214-
3215- return_(bu);
3216-}
3217-
3218-//-------------------------------
3219-class StartBackupCleanUp : public CSRefObject {
3220- bool do_cleanup;
3221- MSBackup *backup;
3222-
3223- public:
3224-
3225- StartBackupCleanUp(): CSRefObject(),
3226- do_cleanup(false){}
3227-
3228- ~StartBackupCleanUp()
3229- {
3230- if (do_cleanup) {
3231- backup->completeBackup();
3232- }
3233- }
3234-
3235- void setCleanUp(MSBackup *bup)
3236- {
3237- backup = bup;
3238- do_cleanup = true;
3239- }
3240-
3241- void cancelCleanUp()
3242- {
3243- do_cleanup = false;
3244- }
3245-
3246-};
3247-
3248-void MSBackup::startBackup(MSDatabase *src_db)
3249-{
3250- CSSyncVector *repo_list;
3251- bool compacting = false;
3252- MSRepository *repo;
3253- StartBackupCleanUp *cleanup;
3254- enter_();
3255-
3256- // Create a cleanup object to handle cleanup
3257- // after a possible exception.
3258- new_(cleanup, StartBackupCleanUp());
3259- push_(cleanup);
3260- cleanup->setCleanUp(this);
3261-
3262- bu_SourceDatabase = src_db;
3263- repo_list = bu_SourceDatabase->getRepositoryList();
3264- // Suspend the compactor before locking the list.
3265- bu_Compactor = bu_SourceDatabase->getCompactorThread();
3266- if (bu_Compactor) {
3267- bu_Compactor->retain();
3268- bu_Compactor->suspend();
3269- }
3270-
3271- // Build the list of repositories to be backed up.
3272- lock_(repo_list);
3273-
3274- new_(bu_BackupList, CSVector(repo_list->size()));
3275- for (uint32_t i = 0; i<repo_list->size(); i++) {
3276- if ((repo = (MSRepository *) repo_list->get(i))) {
3277- if (!repo->isRemovingFP && !repo->mustBeDeleted) {
3278- bu_BackupList->add(RETAIN(repo));
3279- if (repo->initBackup() == REPO_COMPACTING)
3280- compacting = true;
3281-
3282- if (!repo->myRepoHeadSize) {
3283- /* The file has not yet been opened, so the
3284- * garbage count will not be known!
3285- */
3286- MSRepoFile *repo_file;
3287-
3288- //repo->retain();
3289- //unlock_(myRepostoryList);
3290- //push_(repo);
3291- repo_file = repo->openRepoFile();
3292- repo_file->release();
3293- //release_(repo);
3294- //lock_(myRepostoryList);
3295- //goto retry;
3296- }
3297-
3298- bu_size += repo->myRepoFileSize;
3299-
3300- }
3301- }
3302- }
3303-
3304- // Copy the table list to the backup database:
3305- uint32_t next_tab = 0;
3306- MSTable *tab;
3307- while ((tab = bu_SourceDatabase->getNextTable(&next_tab))) {
3308- push_(tab);
3309- bu_Database->addTable(tab->myTableID, tab->myTableName->getCString(), 0, false);
3310- release_(tab);
3311- }
3312- unlock_(repo_list);
3313-
3314- // Copy over any physical PBMS system tables.
3315- PBMSSystemTables::transferSystemTables(RETAIN(bu_Database), RETAIN(bu_SourceDatabase));
3316-
3317- // Load the system tables into the backup database. This will
3318- // initialize the database for cloud storage if required.
3319- PBMSSystemTables::loadSystemTables(RETAIN(bu_Database));
3320-
3321- // Set the cloud backup info.
3322- bu_Database->myBlobCloud->cl_setBackupInfo(RETAIN(bu_info));
3323-
3324-
3325- // Set the backup number in the pbms_variable tabe. (This is a hidden value.)
3326- // This value is used in case a drag and drop restore was done. When a data base is
3327- // first loaded this value is checked and if it is not zero then the backup record
3328- // will be read and any used to recover any BLOBs.
3329- //
3330- char value[20];
3331- snprintf(value, 20, "%"PRIu32"", bu_info->getBackupRefId());
3332- MSVariableTable::setVariable(RETAIN(bu_Database), BACKUP_NUMBER_VAR, value);
3333-
3334- // Once the repositories are locked the compactor can be restarted
3335- // unless it is in the process of compacting a repository that is
3336- // being backed up.
3337- if (bu_Compactor && !compacting) {
3338- bu_Compactor->resume();
3339- bu_Compactor->release();
3340- bu_Compactor = NULL;
3341- }
3342-
3343- // Suspend the transaction writer while the backup is running.
3344- MSTransactionManager::suspend(true);
3345- bu_TransactionManagerSuspended = true;
3346-
3347- // Start the backup daemon thread.
3348- bu_ID = bu_start_time = time(NULL);
3349- start();
3350-
3351- cleanup->cancelCleanUp();
3352- release_(cleanup);
3353-
3354- exit_();
3355-}
3356-
3357-void MSBackup::completeBackup()
3358-{
3359- if (bu_TransactionManagerSuspended) {
3360- MSTransactionManager::resume();
3361- bu_TransactionManagerSuspended = false;
3362- }
3363-
3364- if (bu_BackupList) {
3365- MSRepository *repo;
3366-
3367- while (bu_BackupList->size()) {
3368- repo = (MSRepository *) bu_BackupList->take(0);
3369- if (repo) {
3370- repo->backupCompleted();
3371- repo->release();
3372- }
3373- }
3374- bu_BackupList->release();
3375- bu_BackupList = NULL;
3376- }
3377-
3378- if (bu_Compactor) {
3379- bu_Compactor->resume();
3380- bu_Compactor->release();
3381- bu_Compactor = NULL;
3382- }
3383-
3384- if (bu_Database) {
3385- if (bu_State == BU_COMPLETED)
3386- bu_Database->releaseBackupDatabase();
3387- else
3388- MSDatabase::dropDatabase(bu_Database);
3389-
3390- bu_Database = NULL;
3391- }
3392-
3393- if (bu_SourceDatabase){
3394- if (bu_State == BU_COMPLETED)
3395- bu_info->backupCompleted(bu_SourceDatabase);
3396- else
3397- bu_info->backupTerminated(bu_SourceDatabase);
3398-
3399- bu_SourceDatabase = NULL;
3400- bu_info->release();
3401- bu_info = NULL;
3402- }
3403-
3404- bu_BackupRunning = false;
3405-}
3406-
3407-bool MSBackup::doWork()
3408-{
3409- enter_();
3410- try_(a) {
3411- CSMutex *my_lock;
3412- MSRepository *src_repo, *dst_repo;
3413- MSRepoFile *src_file, *dst_file;
3414- off64_t src_offset, prev_offset;
3415- uint16_t head_size;
3416- uint64_t blob_size, blob_data_size;
3417- CSStringBuffer *head;
3418- MSRepoPointersRec ptr;
3419- uint32_t table_ref_count;
3420- uint32_t blob_ref_count;
3421- int ref_count;
3422- size_t ref_size;
3423- uint32_t auth_code;
3424- uint32_t tab_id;
3425- uint64_t blob_id;
3426- MSOpenTable *otab;
3427- uint32_t src_repo_id;
3428- uint8_t status;
3429- uint8_t blob_storage_type;
3430- uint16_t tab_index;
3431- uint32_t mod_time;
3432- char *transferBuffer;
3433- CloudKeyRec cloud_key;
3434-
3435-
3436- bu_BackupRunning = true;
3437- bu_State = BU_RUNNING;
3438-
3439- /*
3440- // For testing:
3441- {
3442- int blockit = 0;
3443- myWaitTime = 5 * 1000; // Time in milli-seconds
3444- while (blockit)
3445- return_(true);
3446- }
3447- */
3448-
3449- transferBuffer = (char*) cs_malloc(MS_BACKUP_BUFFER_SIZE);
3450- push_ptr_(transferBuffer);
3451-
3452- new_(head, CSStringBuffer(100));
3453- push_(head);
3454-
3455- src_repo = (MSRepository*)bu_BackupList->get(0);
3456- while (src_repo && !myMustQuit) {
3457- src_offset = 0;
3458- src_file = src_repo->openRepoFile();
3459- push_(src_file);
3460-
3461- dst_repo = bu_Database->lockRepo(src_repo->myRepoFileSize - src_repo->myGarbageCount);
3462- frompool_(dst_repo);
3463- dst_file = dst_repo->openRepoFile();
3464- push_(dst_file);
3465-
3466- src_repo_id = src_repo->myRepoID;
3467- src_offset = src_repo->myRepoHeadSize;
3468- prev_offset = 0;
3469- while (src_offset < src_repo->myRepoFileSize) {
3470- retry_read:
3471-
3472- bu_completed += src_offset - prev_offset;
3473- prev_offset = src_offset;
3474- suspended();
3475-
3476- if (myMustQuit)
3477- break;
3478-
3479- // A lock is required here because references and dereferences to the
3480- // BLOBs can result in the repository record being updated while
3481- // it is being copied.
3482- my_lock = &src_repo->myRepoLock[src_offset % CS_REPO_REC_LOCK_COUNT];
3483- lock_(my_lock);
3484- head->setLength(src_repo->myRepoBlobHeadSize);
3485- if (src_file->read(head->getBuffer(0), src_offset, src_repo->myRepoBlobHeadSize, 0) < src_repo->myRepoBlobHeadSize) {
3486- unlock_(my_lock);
3487- break;
3488- }
3489-
3490- ptr.rp_chars = head->getBuffer(0);
3491- ref_size = CS_GET_DISK_1(ptr.rp_head->rb_ref_size_1);
3492- ref_count = CS_GET_DISK_2(ptr.rp_head->rb_ref_count_2);
3493- head_size = CS_GET_DISK_2(ptr.rp_head->rb_head_size_2);
3494- blob_size = CS_GET_DISK_6(ptr.rp_head->rb_blob_repo_size_6);
3495- blob_data_size = CS_GET_DISK_6(ptr.rp_head->rb_blob_data_size_6);
3496- auth_code = CS_GET_DISK_4(ptr.rp_head->rb_auth_code_4);
3497- status = CS_GET_DISK_1(ptr.rp_head->rb_status_1);
3498- mod_time = CS_GET_DISK_4(ptr.rp_head->rb_mod_time_4);
3499-
3500- blob_storage_type = CS_GET_DISK_1(ptr.rp_head->rb_storage_type_1);
3501- if (blob_storage_type == MS_CLOUD_STORAGE) {
3502- MSRepoFile::getBlobKey(ptr.rp_head, &cloud_key);
3503- }
3504-
3505- // If the BLOB was modified after the start of the backup
3506- // then set the mod time to the backup time to ensure that
3507- // a backup for update will work correctly.
3508- if (mod_time > bu_start_time)
3509- CS_SET_DISK_4(ptr.rp_head->rb_mod_time_4, bu_start_time);
3510-
3511- // If the BLOB was moved during the time of this backup then copy
3512- // it to the backup location as a referenced BLOB.
3513- if ((status == MS_BLOB_MOVED) && (bu_ID == (uint32_t) CS_GET_DISK_4(ptr.rp_head->rb_backup_id_4))) {
3514- status = MS_BLOB_REFERENCED;
3515- CS_SET_DISK_1(ptr.rp_head->rb_status_1, status);
3516- }
3517-
3518- // sanity check
3519- if ((blob_data_size == 0) || ref_count <= 0 || ref_size == 0 ||
3520- head_size < src_repo->myRepoBlobHeadSize + ref_count * ref_size ||
3521- !VALID_BLOB_STATUS(status)) {
3522- /* Can't be true. Assume this is garbage! */
3523- src_offset++;
3524- unlock_(my_lock);
3525- continue;
3526- }
3527-
3528-
3529- if ((status == MS_BLOB_REFERENCED) || (status == MS_BLOB_MOVED)) {
3530- head->setLength(head_size);
3531- if (src_file->read(head->getBuffer(0) + src_repo->myRepoBlobHeadSize, src_offset + src_repo->myRepoBlobHeadSize, head_size - src_repo->myRepoBlobHeadSize, 0) != (head_size- src_repo->myRepoBlobHeadSize)) {
3532- unlock_(my_lock);
3533- break;
3534- }
3535-
3536- table_ref_count = 0;
3537- blob_ref_count = 0;
3538-
3539- // Loop through all the references removing temporary references
3540- // and counting table and blob references.
3541-
3542- ptr.rp_chars = head->getBuffer(0) + src_repo->myRepoBlobHeadSize;
3543- for (int count = 0; count < ref_count; count++) {
3544- switch (CS_GET_DISK_2(ptr.rp_ref->rr_type_2)) {
3545- case MS_BLOB_FREE_REF:
3546- break;
3547- case MS_BLOB_TABLE_REF:
3548- // Unlike the compactor, table refs are not checked because
3549- // they do not yet exist in the backup database.
3550- table_ref_count++;
3551- break;
3552- case MS_BLOB_DELETE_REF:
3553- // These are temporary references from the TempLog file.
3554- // They are not copied to the backup.
3555- CS_SET_DISK_2(ptr.rp_ref->rr_type_2, MS_BLOB_FREE_REF);
3556- break;
3557- default:
3558- // Must be a BLOB reference
3559-
3560- tab_index = CS_GET_DISK_2(ptr.rp_blob_ref->er_table_2);
3561- if (tab_index && (tab_index <= ref_count)) {
3562- // Only committed references are backed up.
3563- if (IS_COMMITTED(CS_GET_DISK_8(ptr.rp_blob_ref->er_blob_ref_id_8))) {
3564- MSRepoTableRefPtr tab_ref;
3565- tab_ref = (MSRepoTableRefPtr) (head->getBuffer(0) + src_repo->myRepoBlobHeadSize + (tab_index-1) * ref_size);
3566- if (CS_GET_DISK_2(tab_ref->rr_type_2) == MS_BLOB_TABLE_REF)
3567- blob_ref_count++;
3568- } else {
3569- CS_SET_DISK_2(ptr.rp_ref->rr_type_2, MS_BLOB_FREE_REF);
3570- }
3571-
3572- } else {
3573- /* Can't be true. Assume this is garbage! */
3574- src_offset++;
3575- unlock_(my_lock);
3576- goto retry_read;
3577- }
3578- break;
3579- }
3580- ptr.rp_chars += ref_size;
3581- }
3582-
3583-
3584- // If there are still blob references then the record needs to be backed up.
3585- if (table_ref_count && blob_ref_count) {
3586-
3587- off64_t dst_offset;
3588-
3589- dst_offset = dst_repo->myRepoFileSize;
3590-
3591- /* Write the header. */
3592- dst_file->write(head->getBuffer(0), dst_offset, head_size);
3593-
3594- /* Copy the BLOB over: */
3595- if (blob_storage_type == MS_CLOUD_STORAGE) {
3596- bu_Database->myBlobCloud->cl_backupBLOB(&cloud_key);
3597- } else
3598- CSFile::transfer(RETAIN(dst_file), dst_offset + head_size, RETAIN(src_file), src_offset + head_size, blob_size, transferBuffer, MS_BACKUP_BUFFER_SIZE);
3599-
3600- /* Update the references: */
3601- ptr.rp_chars = head->getBuffer(0) + src_repo->myRepoBlobHeadSize;
3602- for (int count = 0; count < ref_count; count++) {
3603- switch (CS_GET_DISK_2(ptr.rp_ref->rr_type_2)) {
3604- case MS_BLOB_FREE_REF:
3605- case MS_BLOB_DELETE_REF:
3606- break;
3607- case MS_BLOB_TABLE_REF:
3608- tab_id = CS_GET_DISK_4(ptr.rp_tab_ref->tr_table_id_4);
3609- blob_id = CS_GET_DISK_6(ptr.rp_tab_ref->tr_blob_id_6);
3610-
3611- if ((otab = MSTableList::getOpenTableByID(bu_Database->myDatabaseID, tab_id))) {
3612- frompool_(otab);
3613- otab->getDBTable()->setBlobHandle(otab, blob_id, dst_repo->myRepoID, dst_offset, blob_size, head_size, auth_code);
3614-//CSException::throwException(CS_CONTEXT, MS_ERR_NOT_IMPLEMENTED, "What if an error ocurred here!");
3615-
3616- backtopool_(otab);
3617- }
3618- break;
3619- default:
3620- break;
3621- }
3622- ptr.rp_chars += ref_size;
3623- }
3624-
3625- dst_repo->myRepoFileSize += head_size + blob_size;
3626- }
3627- }
3628- unlock_(my_lock);
3629- src_offset += head_size + blob_size;
3630- }
3631- bu_completed += src_offset - prev_offset;
3632-
3633- // close the destination repository and cleanup.
3634- release_(dst_file);
3635- backtopool_(dst_repo);
3636- release_(src_file);
3637-
3638- // release the source repository and get the next one in the list.
3639- src_repo->backupCompleted();
3640- bu_BackupList->remove(0);
3641-
3642- src_repo = (MSRepository*)bu_BackupList->get(0);
3643- }
3644-
3645- release_(head);
3646- release_(transferBuffer);
3647- if (myMustQuit)
3648- bu_State = BU_TERMINATED;
3649- else
3650- bu_State = BU_COMPLETED;
3651-
3652- }
3653-
3654- catch_(a) {
3655- logException();
3656- }
3657-
3658- cont_(a);
3659- completeBackup();
3660- myMustQuit = true;
3661- return_(true);
3662-}
3663-
3664-void *MSBackup::completeWork()
3665-{
3666- if (bu_SourceDatabase || bu_BackupList || bu_Compactor || bu_info) {
3667- // We shouldn't be here
3668- CSException::throwException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, "MSBackup::completeBackup() not called");
3669- if (bu_SourceDatabase) {
3670- bu_SourceDatabase->release();
3671- bu_SourceDatabase = NULL;
3672- }
3673-
3674- if (bu_BackupList) {
3675- bu_BackupList->release();
3676- bu_BackupList = NULL;
3677- }
3678-
3679-
3680- if (bu_Compactor) {
3681- bu_Compactor->release();
3682- bu_Compactor = NULL;
3683- }
3684-
3685-
3686- if (bu_info) {
3687- bu_info->release();
3688- bu_info = NULL;
3689- }
3690-
3691- }
3692- return NULL;
3693-}
3694
3695=== removed file 'plugin/pbms/src/backup_ms.h'
3696--- plugin/pbms/src/backup_ms.h 2011-03-14 05:40:28 +0000
3697+++ plugin/pbms/src/backup_ms.h 1970-01-01 00:00:00 +0000
3698@@ -1,186 +0,0 @@
3699-/* Copyright (C) 2009 PrimeBase Technologies GmbH, Germany
3700- *
3701- * PrimeBase Media Stream for MySQL
3702- *
3703- * This program is free software; you can redistribute it and/or modify
3704- * it under the terms of the GNU General Public License as published by
3705- * the Free Software Foundation; either version 2 of the License, or
3706- * (at your option) any later version.
3707- *
3708- * This program is distributed in the hope that it will be useful,
3709- * but WITHOUT ANY WARRANTY; without even the implied warranty of
3710- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
3711- * GNU General Public License for more details.
3712- *
3713- * You should have received a copy of the GNU General Public License
3714- * along with this program; if not, write to the Free Software
3715- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
3716- *
3717- * Barry Leslie
3718- *
3719- * 2009-05-29
3720- *
3721- * H&G2JCtL
3722- *
3723- * Repository backup.
3724- *
3725- */
3726-
3727-#pragma once
3728-#ifndef _BACKUP_MS_H_
3729-#define _BACKUP_MS_H_
3730-
3731-#include <inttypes.h>
3732-
3733-class MSDatabase;
3734-
3735-class MSBackupInfo : public CSRefObject {
3736- friend class StartDumpCleanUp;
3737- friend class InsertRowCleanUp;
3738-
3739- private:
3740- static uint32_t gMaxInfoRef;
3741- static CSSyncSparseArray *gBackupInfo;
3742-
3743- friend class MSBackupTable;
3744- friend class MSBackup;
3745-
3746- private:
3747- uint32_t backupRefId;
3748- CSString *db_name;
3749- uint32_t db_id;
3750- time_t startTime;
3751- time_t completionTime;
3752- bool dump;
3753- bool isRunning;
3754- CSString *backupLocation;
3755- uint32_t cloudRef;
3756- uint32_t cloudBackupNo;
3757-
3758-public:
3759-
3760- static void startUp()
3761- {
3762- new_(gBackupInfo, CSSyncSparseArray(5));
3763- gMaxInfoRef = 0;
3764- }
3765-
3766- static void shutDown()
3767- {
3768- if (gBackupInfo) {
3769- gBackupInfo->clear();
3770- gBackupInfo->release();
3771- gBackupInfo = NULL;
3772- }
3773- }
3774-
3775-
3776- static MSBackupInfo *findBackupInfo(uint32_t in_backupRefId)
3777- {
3778- MSBackupInfo *info;
3779- enter_();
3780-
3781- lock_(gBackupInfo);
3782-
3783- info = (MSBackupInfo *) gBackupInfo->get(in_backupRefId);
3784- if (info)
3785- info->retain();
3786- unlock_(gBackupInfo);
3787- return_(info);
3788- }
3789-
3790- static MSBackupInfo *getBackupInfo(uint32_t in_backupRefId)
3791- {
3792- MSBackupInfo *info = findBackupInfo(in_backupRefId);
3793- if (!info) {
3794- enter_();
3795- char msg[80];
3796- snprintf(msg, 80, "Backup info with reference ID %"PRIu32" not found", in_backupRefId);
3797- CSException::throwException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, msg);
3798- outer_();
3799- }
3800- return info;
3801- }
3802-
3803-
3804- MSBackupInfo(uint32_t id, const char *name, uint32_t db_id, time_t start, time_t end, bool isDump, const char *location, uint32_t cloudRef_arg, uint32_t cloudBackupNo_arg );
3805- ~MSBackupInfo();
3806-
3807- uint32_t getBackupRefId() { return backupRefId;}
3808-
3809- const char *getName(){ return db_name->getCString(); }
3810-
3811- uint32_t getDatabaseId() { return db_id;}
3812-
3813- time_t getStart(){ return startTime;}
3814-
3815- time_t getEnd(){ return completionTime;}
3816-
3817- bool isDump(){ return dump;}
3818-
3819- bool isBackupRunning(){ return isRunning;}
3820-
3821- const char *getLocation() { return (backupLocation)?backupLocation->getCString():NULL; }
3822-
3823- uint32_t getcloudRef(){ return cloudRef;}
3824- void setcloudRef(uint32_t no){ cloudRef = no;}
3825-
3826- uint32_t getcloudBackupNo(){ return cloudBackupNo;}
3827- void setcloudBackupNo(uint32_t no){ cloudBackupNo = no;}
3828-
3829- static MSBackupInfo *startDump(MSDatabase *db, uint32_t cloud_ref, uint32_t backup_no);
3830-
3831- void startBackup(MSDatabase *pbms_db);
3832- void backupCompleted(MSDatabase *db);
3833- void backupTerminated(MSDatabase *db);
3834-};
3835-
3836-
3837-class MSDatabase;
3838-class MSOpenSystemTable;
3839-
3840-class MSBackup :public CSDaemon {
3841-
3842-public:
3843-
3844- MSBackup();
3845- ~MSBackup(){} // Do nothing here because 'self' will no longer be valid, use completeWork().
3846-
3847- virtual bool doWork();
3848-
3849- virtual void *completeWork();
3850-
3851- void startBackup(MSDatabase *src_db);
3852- uint64_t getBackupSize() { return bu_size;}
3853- uint64_t getBackupCompletedSize() { return bu_completed;}
3854- bool isRunning() { return bu_BackupRunning;}
3855- int getStatus() { return (bu_BackupRunning)?0:bu_State;}
3856- uint32_t backupID() { return bu_ID;}
3857-
3858- static MSBackup* newMSBackup(MSBackupInfo *backup_info);
3859-
3860- friend class StartBackupCleanUp;
3861-private:
3862- void completeBackup();
3863-
3864- MSBackupInfo *bu_info;
3865-
3866- CSVector *bu_BackupList;
3867- CSDaemon *bu_Compactor;
3868- bool bu_BackupRunning;
3869- enum {BU_RUNNING = -1, BU_COMPLETED = 0, BU_TERMINATED = 1} bu_State;
3870-
3871- MSDatabase *bu_SourceDatabase; // The source database.
3872- MSDatabase *bu_Database; // The destination database.
3873- MSOpenSystemTable *bu_dst_dump; // The source database's pbms_dump.
3874- MSOpenSystemTable *bu_src_dump; // The source database's pbms_dump.
3875- uint64_t bu_size; // The total size of the data to be backed up.
3876- uint64_t bu_completed; // The amount of data that has been backed up so far.
3877-
3878- uint32_t bu_ID;
3879- uint32_t bu_start_time;
3880-
3881- bool bu_TransactionManagerSuspended;
3882-};
3883-
3884-#endif // _BACKUP_MS_H_
3885
3886=== added directory 'plugin/pbms/src/cloud'
3887=== renamed file 'plugin/pbms/src/cloud_ms.cc' => 'plugin/pbms/src/cloud/cloud_ms.cc'
3888--- plugin/pbms/src/cloud_ms.cc 2011-04-20 22:18:30 +0000
3889+++ plugin/pbms/src/cloud/cloud_ms.cc 2011-06-01 23:57:43 +0000
3890@@ -20,15 +20,13 @@
3891 *
3892 */
3893
3894+#include "includes/pbms_config.h"
3895+
3896 #ifdef DRIZZLED
3897-#include <config.h>
3898-#include <drizzled/common.h>
3899-#include <drizzled/session.h>
3900 #include <drizzled/table.h>
3901 #include <drizzled/message/table.pb.h>
3902 #include <drizzled/charset.h>
3903 #include <drizzled/table_proto.h>
3904-#include <drizzled/session.h>
3905 #include <drizzled/field.h>
3906 #endif
3907
3908@@ -49,14 +47,11 @@
3909 #include "cslib/CSEncode.h"
3910 #include "cslib/CSS3Protocol.h"
3911
3912-#include "backup_ms.h"
3913 #include "cloud_ms.h"
3914
3915-CSSyncSparseArray *MSCloudInfo::gCloudInfo;
3916-uint32_t MSCloudInfo::gMaxInfoRef;
3917+#include "lib/pbmslib.h" // Needed for MS_CLOUD_STORAGE
3918+#include "database/database_ms.h"
3919
3920-uint32_t CloudDB::gKeyIndex;
3921-CSMutex CloudDB::gCloudKeyLock;
3922
3923 //==============================
3924 MSCloudInfo::MSCloudInfo(uint32_t id,
3925@@ -94,24 +89,50 @@
3926 }
3927
3928 //-------------------------------
3929+void MSCloudInfo::setServer(const char *server)
3930+{
3931+ s3Prot->s3_setServer(server);
3932+}
3933+
3934+//-------------------------------
3935 const char *MSCloudInfo::getBucket()
3936 {
3937 return bucket->getCString();
3938 }
3939
3940 //-------------------------------
3941+void MSCloudInfo::setBucket(const char *bucket_arg)
3942+{
3943+ if (bucket)
3944+ bucket->release();
3945+ bucket = CSString::newString(bucket_arg);
3946+}
3947+
3948+//-------------------------------
3949 const char *MSCloudInfo::getPublicKey()
3950 {
3951 return s3Prot->s3_getPublicKey();
3952 }
3953
3954 //-------------------------------
3955+void MSCloudInfo::setPublicKey(const char *key)
3956+{
3957+ s3Prot->s3_setPublicKey(key);
3958+}
3959+
3960+//-------------------------------
3961 const char *MSCloudInfo::getPrivateKey()
3962 {
3963 return s3Prot->s3_getPrivateKey();
3964 }
3965
3966 //-------------------------------
3967+void MSCloudInfo::setPrivateKey(const char *key)
3968+{
3969+ s3Prot->s3_setPrivateKey(key);
3970+}
3971+
3972+//-------------------------------
3973 CSString *MSCloudInfo::getSignature(const char *key, const char *content_type, uint32_t *s3AuthorizationTime)
3974 {
3975 return s3Prot->s3_getAuthorization(bucket->getCString(), key, content_type, s3AuthorizationTime);
3976@@ -183,20 +204,18 @@
3977 }
3978
3979 //==============================
3980-CloudDB::CloudDB(uint32_t db_id):
3981+CloudDB::CloudDB(MSDatabase *db_backref):
3982 dfltCloudRefId(0),
3983 keep_alive(5 * 60),// default URL keep alive in seconds.
3984- blob_recovery_no(0),
3985- blob_db_id(db_id),
3986- isBackup(false),
3987- backupInfo(NULL),
3988- backupCloud(NULL),
3989 clObjectKey(NULL)
3990 {
3991 enter_();
3992
3993+ cl_db = db_backref;
3994 new_(clObjectKey, CSStringBuffer());
3995- clObjectKey->setLength(base_key_size);
3996+ clObjectKey->setLength(CloudObjectKey::base_key_size);
3997+
3998+ new_(clCloudInfo, CSSyncSparseArray(5));
3999
4000 exit_();
4001 }
4002@@ -205,234 +224,120 @@
4003 CloudDB::~CloudDB()
4004 {
4005
4006- if (backupInfo)
4007- backupInfo->release();
4008-
4009- if (backupCloud)
4010- backupCloud->release();
4011-
4012 if (clObjectKey)
4013 clObjectKey->release();
4014
4015-}
4016-//-------------------------------
4017-MSBackupInfo *CloudDB::cl_getBackupInfo()
4018+ if (clCloudInfo) {
4019+ clCloudInfo->clear();
4020+ clCloudInfo->release();
4021+ }
4022+
4023+}
4024+
4025+//-------------------------------
4026+MSCloudInfo *CloudDB::getCloudInfo(uint32_t in_cloudRefId)
4027+{
4028+ MSCloudInfo *info;
4029+ enter_();
4030+
4031+ lock_(this);
4032+
4033+ info = (MSCloudInfo *) clCloudInfo->get(in_cloudRefId);
4034+ if (!info) {
4035+ char msg[80];
4036+ snprintf(msg, 80, "Cloud info with reference ID %"PRIu32" not found", in_cloudRefId);
4037+ CSException::throwException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, msg);
4038+ }
4039+ info->retain();
4040+ unlock_(this);
4041+ return_(info);
4042+}
4043+
4044+//-------------------------------
4045+void CloudDB::setDefaultCloudRef(uint32_t dflt)
4046 {
4047- if (backupInfo)
4048- backupInfo->retain();
4049-
4050- return backupInfo;
4051-}
4052-
4053-//-------------------------------
4054-void CloudDB::cl_clearBackupInfo(){ backupInfo->release(); backupInfo = NULL;}
4055-
4056-//-------------------------------
4057-void CloudDB::cl_createDB()
4058+ dfltCloudRefId = dflt;
4059+ cl_db->setBlobType((dflt)?MS_CLOUD_STORAGE:MS_STANDARD_STORAGE);
4060+}
4061+
4062+//-------------------------------
4063+uint32_t CloudDB::newCloudInfo(uint32_t id, const char *server, const char *bucket, const char *publicKey, const char *privateKey, bool isDefault )
4064+{
4065+ MSCloudInfo *info;
4066+ enter_();
4067+
4068+ if (id == 0)
4069+ id = clCloudInfo->maxIndex() + 1;
4070+
4071+ new_(info, MSCloudInfo(id, server, bucket, publicKey, privateKey));
4072+
4073+ push_(info);
4074+ lock_(this);
4075+ clCloudInfo->set(id, info);
4076+ unlock_(this);
4077+ pop_(info);
4078+
4079+ if (isDefault)
4080+ setDefaultCloudRef(id);
4081+
4082+ return_(id);
4083+}
4084+
4085+//-------------------------------
4086+void CloudDB::deleteCloudInfo(uint32_t id)
4087+{
4088+ enter_();
4089+ lock_(this);
4090+ if (id == dfltCloudRefId)
4091+ setDefaultCloudRef(0);
4092+
4093+ MSCloudInfo *info = (MSCloudInfo*) clCloudInfo->take(id);
4094+ if (info) info->release();
4095+ unlock_(this);
4096+ exit_();
4097+}
4098+
4099+//-------------------------------
4100+void CloudDB::getNewKey(CloudKeyPtr key)
4101+{
4102+ enter_();
4103+ lock_(this);
4104+
4105+ key->creation_time = time(NULL);
4106+ if (keyIndex == 0) keyIndex++;
4107+ key->ref_index = keyIndex++;
4108+ key->cloud_ref = dfltCloudRefId;
4109+
4110+ unlock_(this);
4111+ exit_();
4112+}
4113+
4114+//-------------------------------
4115+CSString *CloudDB::getObjectKey(CloudKeyPtr key)
4116+{
4117+ CloudObjectKey *objectKey;
4118+ enter_();
4119+
4120+ new_(objectKey, CloudObjectKey(cl_db->getDatabaseID()));
4121+ push_(objectKey);
4122+
4123+ objectKey->setObjectKey(key);
4124+
4125+ CSString *str = CSString::newString(objectKey->getCString());
4126+ release_(objectKey);
4127+
4128+ return_(str);
4129+}
4130+
4131+//-------------------------------
4132+void CloudDB::createDB()
4133 {
4134 // This is a no-op.
4135 }
4136
4137 //-------------------------------
4138-// Restore all the
4139-void CloudDB::cl_restoreDB()
4140-{
4141- CSVector *list = NULL;
4142- CSString *key = NULL;
4143- CloudObjectKey *src_objectKey = NULL, *dst_objectKey = NULL;
4144- CloudKeyRec cloudKey;
4145- uint32_t src_cloudRef, dst_cloudRef = 0;
4146- MSBackupInfo *backup_info = NULL;
4147- MSCloudInfo *src_cloud = NULL, *dst_cloud = NULL;
4148- enter_();
4149-
4150- if (!blob_recovery_no)
4151- exit_(); // nothing to do.
4152-
4153- backup_info = MSBackupInfo::getBackupInfo(blob_recovery_no);
4154- push_(backup_info);
4155-
4156- src_cloudRef = backup_info->getcloudRef();
4157- src_cloud = MSCloudInfo::getCloudInfo(src_cloudRef);
4158- push_(src_cloud);
4159-
4160- new_(dst_objectKey, CloudObjectKey(blob_db_id));
4161- push_(dst_objectKey);
4162-
4163- // Get the key for the backup BLOB
4164- new_(src_objectKey, CloudObjectKey(blob_db_id));
4165- push_(src_objectKey);
4166- src_objectKey->setObjectKey(NULL, backup_info->getcloudBackupNo(), backup_info->getDatabaseId());
4167-
4168- // Get a list of all the BLOBs that were backed up.
4169- list = src_cloud->list(src_objectKey->getCString());
4170- release_(src_objectKey);
4171- push_(list);
4172-
4173-
4174- // Go through the list copying the keys.
4175- dst_cloudRef = src_cloudRef;
4176- dst_cloud = src_cloud;
4177- dst_cloud->retain();
4178-
4179- push_ref_(dst_cloud); // Push a reference to dst_cloud so that what ever it references will be released.
4180-
4181- while ((key = (CSString*)(list->take(0))) ) {
4182- push_(key);
4183-
4184- // The source key name must be parsed to get its
4185- // destination cloud reference. The destination for
4186- // the BLOBs may not all be in the same cloud.
4187- CloudObjectKey::parseObjectKey(key->getCString(), &cloudKey);
4188-
4189- // Reset the destination cloud if required.
4190- if (cloudKey.cloud_ref != dst_cloudRef) {
4191- if (dst_cloud) {
4192- dst_cloud->release();
4193- dst_cloud = NULL;
4194- }
4195- dst_cloudRef = cloudKey.cloud_ref;
4196- dst_cloud = MSCloudInfo::getCloudInfo(dst_cloudRef);
4197- }
4198-
4199- // Copy the BLOB to the recovered database.
4200- dst_objectKey->setObjectKey(&cloudKey);
4201- src_cloud->copy(RETAIN(dst_cloud), dst_objectKey->getCString(), key->getCString());
4202- release_(key);
4203-
4204- }
4205-
4206- release_(dst_cloud);
4207-
4208- blob_recovery_no = 0;
4209- release_(list);
4210- release_(dst_objectKey);
4211- release_(src_cloud);
4212- release_(backup_info);
4213- exit_();
4214-}
4215-
4216-//-------------------------------
4217-uint32_t CloudDB::cl_getNextBackupNumber(uint32_t cloud_ref)
4218-{
4219- CloudObjectKey *objectKey;
4220- CSVector *list;
4221- uint32_t backup_no = 0, size = 1;
4222- MSCloudInfo *s3Cloud;
4223- enter_();
4224-
4225- s3Cloud = MSCloudInfo::getCloudInfo((cloud_ref)?cloud_ref:dfltCloudRefId);
4226- push_(s3Cloud);
4227-
4228- new_(objectKey, CloudObjectKey(blob_db_id));
4229- push_(objectKey);
4230-
4231- // Find the next available backup number
4232- while (size) {
4233- backup_no++;
4234- objectKey->setObjectKey(NULL, backup_no); // use the key prefix with the backup number for listing.
4235- list = s3Cloud->list(objectKey->getCString(), 1);
4236- size = list->size();
4237- list->release();
4238- }
4239-
4240- release_(objectKey);
4241- release_(s3Cloud);
4242-
4243- return_(backup_no);
4244-}
4245-
4246-//-------------------------------
4247-void CloudDB::cl_backupBLOB(CloudKeyPtr key)
4248-{
4249- CloudObjectKey *src_objectKey, *dst_objectKey;
4250- uint32_t cloudRef, backupNo;
4251- MSCloudInfo *src_cloud = NULL, *dst_cloud = NULL;
4252- enter_();
4253-
4254- ASSERT(backupInfo);
4255-
4256- if ((cloudRef = backupInfo->getcloudRef()) == 0) {
4257- backupInfo->setcloudRef(dfltCloudRefId);
4258- cloudRef = dfltCloudRefId;
4259- }
4260-
4261- if ((backupNo = backupInfo->getcloudBackupNo()) == 0) {
4262- backupNo = cl_getNextBackupNumber(cloudRef);
4263- backupInfo->setcloudBackupNo(backupNo);
4264- }
4265-
4266- // Set the source object's key
4267- new_(src_objectKey, CloudObjectKey(blob_db_id));
4268- push_(src_objectKey);
4269- src_objectKey->setObjectKey(key);
4270-
4271- // Set the destination object's key
4272- new_(dst_objectKey, CloudObjectKey(blob_db_id));
4273- push_(dst_objectKey);
4274- dst_objectKey->setObjectKey(key, backupNo);
4275-
4276- // Get the source cloud
4277- src_cloud = MSCloudInfo::getCloudInfo((key->cloud_ref)?key->cloud_ref:dfltCloudRefId);
4278- push_(src_cloud);
4279-
4280- // Copy the object to the destination cloud
4281- dst_cloud = MSCloudInfo::getCloudInfo(cloudRef);
4282- src_cloud->copy(dst_cloud, dst_objectKey->getCString(), src_objectKey->getCString());
4283-
4284- release_(src_cloud);
4285- release_(dst_objectKey);
4286- release_(src_objectKey);
4287- exit_();
4288-}
4289-
4290-//-------------------------------
4291-void CloudDB::cl_restoreBLOB(CloudKeyPtr key, uint32_t backup_db_id)
4292-{
4293- CloudObjectKey *src_objectKey, *dst_objectKey;
4294- uint32_t cloudRef, backupNo;
4295- MSCloudInfo *src_cloud = NULL, *dst_cloud = NULL;
4296- enter_();
4297-
4298- ASSERT(backupInfo);
4299-
4300- if ((cloudRef = backupInfo->getcloudRef()) == 0) {
4301- backupInfo->setcloudRef(dfltCloudRefId);
4302- cloudRef = dfltCloudRefId;
4303- }
4304-
4305- if ((backupNo = backupInfo->getcloudBackupNo()) == 0) {
4306- backupNo = cl_getNextBackupNumber(cloudRef);
4307- backupInfo->setcloudBackupNo(backupNo);
4308- }
4309-
4310- // Set the source object's key
4311- new_(src_objectKey, CloudObjectKey(backup_db_id));
4312- push_(src_objectKey);
4313- src_objectKey->setObjectKey(key, backupNo);
4314-
4315- // Set the destination object's key
4316- new_(dst_objectKey, CloudObjectKey(blob_db_id));
4317- push_(dst_objectKey);
4318- dst_objectKey->setObjectKey(key);
4319-
4320- // Get the source cloud
4321- src_cloud = MSCloudInfo::getCloudInfo(cloudRef);
4322- push_(src_cloud);
4323-
4324- // Copy the object to the destination cloud
4325- dst_cloud = MSCloudInfo::getCloudInfo((key->cloud_ref)?key->cloud_ref:dfltCloudRefId);
4326- src_cloud->copy(dst_cloud, dst_objectKey->getCString(), src_objectKey->getCString());
4327-
4328- release_(src_cloud);
4329- release_(dst_objectKey);
4330- release_(src_objectKey);
4331- exit_();
4332-}
4333-
4334-//-------------------------------
4335 // Drop database deletes all objects with the database key prefix
4336-void CloudDB::cl_dropDB()
4337+void CloudDB::dropDB()
4338 {
4339 CSVector *list;
4340 CSString *key;
4341@@ -442,30 +347,20 @@
4342 const char *key_str;
4343
4344 enter_();
4345- new_(objectKey, CloudObjectKey(blob_db_id));
4346+ new_(objectKey, CloudObjectKey(cl_db->getDatabaseID()));
4347 push_(objectKey);
4348
4349- lock_(MSCloudInfo::gCloudInfo);
4350+ lock_(this);
4351
4352- if (isBackup) {
4353- uint32_t backup_no;
4354- if (backupInfo && (backup_no = backupInfo->getcloudBackupNo())) {
4355- objectKey->setObjectKey(NULL, backup_no); // use the key prefix for the backup for listing.
4356- if ((s3Cloud = MSCloudInfo::getCloudInfo(backupInfo->getcloudRef())))
4357- push_(s3Cloud);
4358- }
4359- } else {
4360- objectKey->setObjectKey(); // use the key prefix for listing.
4361- i = 0;
4362- s3Cloud = (MSCloudInfo*)MSCloudInfo::gCloudInfo->itemAt(i++); // <-- unreferenced object
4363- }
4364-
4365+ objectKey->setObjectKey(); // use the key prefix for listing.
4366 key_str = objectKey->getCString();
4367
4368 // For non backup BLOBs all known clouds must be searched
4369 // for possible BLOBs and deleted. The BLOBs belonging to a backup
4370 // will ever only be in one cloud storage location.
4371- while (s3Cloud) {
4372+ i = 0;
4373+ while ((s3Cloud = getCloudInfoAt(i++))) {
4374+ push_(s3Cloud);
4375 list = s3Cloud->list(key_str);
4376 push_(list);
4377
4378@@ -477,20 +372,16 @@
4379 }
4380
4381 release_(list);
4382- if (isBackup) {
4383- release_(s3Cloud); // Only the backup s3Cloud needs to be released.
4384- s3Cloud = NULL;
4385- } else
4386- s3Cloud = (MSCloudInfo*)MSCloudInfo::gCloudInfo->itemAt(i++);// <-- unreferenced object
4387+ release_(s3Cloud);
4388 }
4389
4390- unlock_(MSCloudInfo::gCloudInfo);
4391+ unlock_(this);
4392 release_(objectKey);
4393 exit_();
4394 }
4395
4396 //-------------------------------
4397-void CloudDB::cl_putData(CloudKeyPtr key, CSInputStream *stream, off64_t size)
4398+void CloudDB::putData(CloudKeyPtr key, CSInputStream *stream, off64_t size)
4399 {
4400 CloudObjectKey *objectKey;
4401 MSCloudInfo *s3Cloud;
4402@@ -499,12 +390,12 @@
4403
4404 push_(stream);
4405
4406- new_(objectKey, CloudObjectKey(blob_db_id));
4407+ new_(objectKey, CloudObjectKey(cl_db->getDatabaseID()));
4408 push_(objectKey);
4409
4410 objectKey->setObjectKey(key);
4411
4412- s3Cloud = MSCloudInfo::getCloudInfo((key->cloud_ref)?key->cloud_ref:dfltCloudRefId);
4413+ s3Cloud = getCloudInfo((key->cloud_ref)?key->cloud_ref:dfltCloudRefId);
4414 push_(s3Cloud);
4415 s3Cloud->send(RETAIN(stream), objectKey->getCString(), size);
4416 release_(s3Cloud);
4417@@ -516,17 +407,17 @@
4418 }
4419
4420 //-------------------------------
4421-off64_t CloudDB::cl_getData(CloudKeyPtr key, char *buffer, off64_t size)
4422+off64_t CloudDB::getData(CloudKeyPtr key, char *buffer, off64_t size)
4423 {
4424 CloudObjectKey *objectKey;
4425 CSStaticMemoryOutputStream *output;
4426 MSCloudInfo *s3Cloud;
4427 enter_();
4428
4429- new_(objectKey, CloudObjectKey(blob_db_id));
4430+ new_(objectKey, CloudObjectKey(cl_db->getDatabaseID()));
4431 push_(objectKey);
4432
4433- s3Cloud = MSCloudInfo::getCloudInfo(key->cloud_ref);
4434+ s3Cloud = getCloudInfo(key->cloud_ref);
4435 push_(s3Cloud);
4436
4437 new_(output, CSStaticMemoryOutputStream((u_char *)buffer, size));
4438@@ -544,16 +435,16 @@
4439 }
4440
4441 //-------------------------------
4442-void CloudDB::cl_deleteData(CloudKeyPtr key)
4443+void CloudDB::deleteData(CloudKeyPtr key)
4444 {
4445 MSCloudInfo *s3Cloud;
4446 CloudObjectKey *objectKey;
4447 enter_();
4448
4449- new_(objectKey, CloudObjectKey(blob_db_id));
4450+ new_(objectKey, CloudObjectKey(cl_db->getDatabaseID()));
4451 push_(objectKey);
4452
4453- s3Cloud = MSCloudInfo::getCloudInfo(key->cloud_ref);
4454+ s3Cloud = getCloudInfo(key->cloud_ref);
4455 push_(s3Cloud);
4456
4457 objectKey->setObjectKey(key);
4458@@ -567,19 +458,19 @@
4459 }
4460
4461 //-------------------------------
4462-CSString *CloudDB::cl_getDataURL(CloudKeyPtr key)
4463+CSString *CloudDB::getDataURL(CloudKeyPtr key)
4464 {
4465 CloudObjectKey *objectKey;
4466 CSString *url;
4467 MSCloudInfo *s3Cloud;
4468 enter_();
4469
4470- new_(objectKey, CloudObjectKey(blob_db_id));
4471+ new_(objectKey, CloudObjectKey(cl_db->getDatabaseID()));
4472 push_(objectKey);
4473
4474 objectKey->setObjectKey(key);
4475
4476- s3Cloud = MSCloudInfo::getCloudInfo(key->cloud_ref);
4477+ s3Cloud = getCloudInfo(key->cloud_ref);
4478 push_(s3Cloud);
4479
4480 url = s3Cloud->getDataURL(objectKey->getCString(), keep_alive);
4481@@ -591,7 +482,7 @@
4482 }
4483
4484 //-------------------------------
4485-CSString *CloudDB::cl_getSignature(CloudKeyPtr key, CSString *content_type_arg, uint32_t *s3AuthorizationTime)
4486+CSString *CloudDB::getSignature(CloudKeyPtr key, CSString *content_type_arg, uint32_t *s3AuthorizationTime)
4487 {
4488 CSString *signature;
4489 CloudObjectKey *objectKey;
4490@@ -599,7 +490,7 @@
4491 MSCloudInfo *s3Cloud;
4492 enter_();
4493
4494- new_(objectKey, CloudObjectKey(blob_db_id));
4495+ new_(objectKey, CloudObjectKey(cl_db->getDatabaseID()));
4496 push_(objectKey);
4497
4498 if (content_type_arg) {
4499@@ -608,7 +499,7 @@
4500 }
4501
4502 objectKey->setObjectKey(key);
4503- s3Cloud = MSCloudInfo::getCloudInfo(key->cloud_ref);
4504+ s3Cloud = getCloudInfo(key->cloud_ref);
4505 push_(s3Cloud);
4506
4507 signature = s3Cloud->getSignature(objectKey->getCString(), content_type, s3AuthorizationTime);
4508
4509=== renamed file 'plugin/pbms/src/cloud_ms.h' => 'plugin/pbms/src/cloud/cloud_ms.h'
4510--- plugin/pbms/src/cloud_ms.h 2011-03-14 05:40:28 +0000
4511+++ plugin/pbms/src/cloud/cloud_ms.h 2011-06-01 23:57:43 +0000
4512@@ -25,79 +25,47 @@
4513 #include "cslib/CSMd5.h"
4514 #include <inttypes.h>
4515
4516-
4517 /* NOTES:
4518 *
4519- * - TODO: If cl_deleteData() fails then the BLOB deletion must fail and be rescheduled to try again
4520+ * - TODO: If deleteData() fails then the BLOB deletion must fail and be rescheduled to try again
4521 * later.
4522 * - TODO: Copying of BLOBs from one database to another needs to be handled. Look for copyBlob() and
4523 * resetBlobHead(). There are 3 cases to handle depending on if the databases involved use
4524- * cload storage.
4525+ * cloud storage.
4526 */
4527
4528 //===============================
4529 class CSS3Protocol;
4530 class MSCloudInfo : public CSRefObject {
4531 private:
4532- static uint32_t gMaxInfoRef;
4533- static CSSyncSparseArray *gCloudInfo;
4534
4535 friend class MSCloudTable;
4536 friend class CloudDB;
4537
4538 private:
4539- uint32_t cloudRefId;
4540+ uint32_t cloudRefId;
4541 CSString *bucket;
4542 CSS3Protocol *s3Prot;
4543+ bool defaultCloudRef;
4544
4545 public:
4546
4547- static void startUp()
4548- {
4549- new_(gCloudInfo, CSSyncSparseArray(5));
4550- gMaxInfoRef = 0;
4551- }
4552-
4553- static void shutDown()
4554- {
4555- if (gCloudInfo) {
4556- gCloudInfo->clear();
4557- gCloudInfo->release();
4558- gCloudInfo = NULL;
4559- }
4560- }
4561-
4562-
4563- static MSCloudInfo *getCloudInfo(uint32_t in_cloudRefId)
4564- {
4565- MSCloudInfo *info;
4566- enter_();
4567-
4568- lock_(gCloudInfo);
4569-
4570- info = (MSCloudInfo *) gCloudInfo->get(in_cloudRefId);
4571- if (!info) {
4572- char msg[80];
4573- snprintf(msg, 80, "Cloud info with reference ID %"PRIu32" not found", in_cloudRefId);
4574- CSException::throwException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, msg);
4575- }
4576- info->retain();
4577- unlock_(gCloudInfo);
4578- return_(info);
4579- }
4580-
4581- MSCloudInfo(uint32_t id, const char *server, const char *bucket, const char *publicKey, const char *privateKey );
4582+ MSCloudInfo(uint32_t id, const char *server, const char *bucket, const char *publicKey, const char *privateKey);
4583 ~MSCloudInfo();
4584
4585 uint32_t getCloudRefId() { return cloudRefId;}
4586
4587 const char *getServer();
4588+ void setServer(const char *server);
4589
4590 const char *getBucket();
4591+ void setBucket(const char *bucket);
4592
4593 const char *getPublicKey();
4594+ void setPublicKey(const char *key);
4595
4596 const char *getPrivateKey();
4597+ void setPrivateKey(const char *key);
4598
4599 CSString *getSignature(const char *key, const char *content_type, uint32_t *s3AuthorizationTime);
4600
4601@@ -124,13 +92,13 @@
4602 //===============================
4603 class CloudObjectKey : public CSStringBuffer
4604 {
4605- uint32_t default_db_id;
4606-
4607+ uint16_t default_db_id;
4608+
4609 public:
4610- CloudObjectKey(uint32_t id): CSStringBuffer(), default_db_id(id){ }
4611+ CloudObjectKey(uint16_t id): CSStringBuffer(), default_db_id(id){ }
4612 ~CloudObjectKey(){}
4613
4614- static const uint32_t base_key_size = 64; // enough space for <db_id>/<backup_id>/<creation_time>/<ref_index>
4615+ static const uint32_t base_key_size = 64; // enough space for <db_id>/<creation_time>/<ref_index>
4616
4617 void setObjectKey(const char *object_key)
4618 {
4619@@ -139,129 +107,83 @@
4620 snprintf(getBuffer(0), length(), "%"PRIu32"/0/%s",default_db_id, object_key);
4621 }
4622
4623- void setObjectKey(CloudKeyPtr key = NULL, uint32_t backup_id = 0, uint32_t db_id = 0)
4624+ void setObjectKey(CloudKeyPtr key = NULL, uint32_t db_id = 0)
4625 {
4626 if (!db_id) db_id = default_db_id;
4627 setLength(base_key_size);
4628
4629 if (key)
4630- snprintf(getBuffer(0), length(), "%"PRIu32"/%"PRIu32"/%"PRIu32".%"PRIu32".%"PRIu32"", db_id, backup_id, key->cloud_ref, key->creation_time, key->ref_index);
4631+ snprintf(getBuffer(0), length(), "%"PRIu32"/%"PRIu32".%"PRIu32".%"PRIu32"", db_id, key->cloud_ref, key->creation_time, key->ref_index);
4632 else
4633- snprintf(getBuffer(0), length(), "%"PRIu32"/%"PRIu32"s/", db_id, backup_id);
4634+ snprintf(getBuffer(0), length(), "%"PRIu32"/", db_id);
4635
4636 }
4637
4638- static void parseObjectKey(const char *object_key, CloudKeyPtr key, uint32_t *backup_id = NULL, uint32_t *db_id = NULL)
4639+ static void parseObjectKey(const char *object_key, CloudKeyPtr key, uint32_t *db_id = NULL)
4640 {
4641 uint32_t v1;
4642
4643- if (!backup_id) backup_id = &v1;
4644 if (!db_id) db_id = &v1;
4645
4646- sscanf(object_key, "%"PRIu32"/%"PRIu32"/%"PRIu32".%"PRIu32".%"PRIu32"", db_id, backup_id, &(key->cloud_ref), &(key->creation_time), &(key->ref_index));
4647+ sscanf(object_key, "%"PRIu32"/%"PRIu32".%"PRIu32".%"PRIu32"", db_id, &(key->cloud_ref), &(key->creation_time), &(key->ref_index));
4648 }
4649 };
4650
4651 //===============================
4652-class MSBackupInfo;
4653-class CloudDB: public CSRefObject {
4654+class MSDatabase;
4655+class CloudDB: public CSSharedRefObject {
4656
4657 private:
4658- static uint32_t gKeyIndex;
4659- static CSMutex gCloudKeyLock;
4660+ CSSyncSparseArray *clCloudInfo;
4661
4662+ uint32_t keyIndex;
4663 uint32_t dfltCloudRefId;
4664
4665 uint32_t keep_alive; // The length of time a redirect URL will remain valid. In seconds.
4666- uint32_t blob_recovery_no; // This is the backup number from which the recovery should be done.
4667- uint32_t blob_db_id;
4668-
4669- bool isBackup;
4670- MSBackupInfo *backupInfo;
4671- MSCloudInfo *backupCloud;
4672-
4673- static const uint32_t base_key_size = 64; // enough space for <db_id>/<backup_id>/<creation_time>/<ref_index>
4674-
4675+ MSDatabase *cl_db; // A back reference to the database that this object belongs to.
4676+
4677 public:
4678 CSStringBuffer *clObjectKey;
4679
4680- CloudDB(uint32_t db_id);
4681+ CloudDB(MSDatabase *db_backref);
4682 ~CloudDB();
4683
4684- void cl_setDefaultCloudRef(uint32_t dflt) { dfltCloudRefId = dflt;}
4685- uint32_t cl_getDefaultCloudRef() { return dfltCloudRefId;}
4686-
4687- MSCloudInfo *cl_getCloudInfo(uint32_t cloudRefId = 0)
4688- {
4689- return MSCloudInfo::getCloudInfo((cloudRefId)?cloudRefId:dfltCloudRefId);
4690- }
4691-
4692- void cl_getNewKey(CloudKeyPtr key)
4693- {
4694- enter_();
4695- lock_(&gCloudKeyLock);
4696-
4697- key->creation_time = time(NULL);
4698- key->ref_index = gKeyIndex++;
4699- key->cloud_ref = dfltCloudRefId;
4700-
4701- unlock_(&gCloudKeyLock);
4702- exit_();
4703- }
4704-
4705- bool cl_mustRecoverBlobs() { return (blob_recovery_no != 0);}
4706-
4707- void cl_setRecoveryNumber(const char *number)
4708- {
4709- blob_recovery_no = atol(number);
4710- }
4711-
4712- const char *cl_getRecoveryNumber()
4713- {
4714- static char number[20];
4715-
4716- snprintf(number, 20, "%"PRIu32"", blob_recovery_no);
4717- return number;
4718- }
4719-
4720- CSString *cl_getObjectKey(CloudKeyPtr key)
4721- {
4722- CloudObjectKey *objectKey;
4723- enter_();
4724-
4725- new_(objectKey, CloudObjectKey(blob_db_id));
4726- push_(objectKey);
4727-
4728- objectKey->setObjectKey(key);
4729-
4730- CSString *str = CSString::newString(objectKey->getCString());
4731- release_(objectKey);
4732-
4733- return_(str);
4734- }
4735-
4736- void cl_setKeepAlive(uint32_t keep_alive_arg) {keep_alive = keep_alive_arg;}
4737-
4738- void cl_createDB();
4739- void cl_dropDB();
4740- void cl_restoreDB();
4741- uint32_t cl_getNextBackupNumber(uint32_t cloud_ref = 0);
4742- bool cl_dbExists();
4743-
4744- // setting backup_blob_no to -1 ensures that if the database is dropped no BLOBs will be deleted.
4745- void cl_setCloudIsBackup(){ isBackup = true;}
4746- void cl_setBackupInfo(MSBackupInfo *info){ backupInfo = info;}
4747- MSBackupInfo *cl_getBackupInfo();
4748- void cl_clearBackupInfo();
4749-
4750- void cl_backupBLOB(CloudKeyPtr key);
4751- void cl_restoreBLOB(CloudKeyPtr key, uint32_t backup_db_id);
4752-
4753- void cl_putData( CloudKeyPtr key, CSInputStream *stream, off64_t size);
4754- off64_t cl_getData(CloudKeyPtr key, char *data, off64_t size);
4755- CSString *cl_getDataURL(CloudKeyPtr key);
4756- void cl_deleteData(CloudKeyPtr key);
4757- CSString *cl_getSignature(CloudKeyPtr key, CSString *content_type, uint32_t *s3AuthorizationTime);
4758+ bool isLoaded() { return (clCloudInfo->size() > 0); }
4759+ bool isValidID(uint32_t info_id) { return (clCloudInfo->get(info_id) != NULL); }
4760+
4761+ uint32_t getCloudInfoIndexForID(uint32_t id) { return clCloudInfo->getIndex(id);}
4762+
4763+ MSCloudInfo *getCloudInfoAt(uint32_t index)
4764+ {
4765+ MSCloudInfo *info = (MSCloudInfo *)clCloudInfo->itemAt(index);
4766+ if (!info)
4767+ return NULL;
4768+ return RETAIN(info);
4769+ }
4770+
4771+ MSCloudInfo *getCloudInfo(uint32_t in_cloudRefId);
4772+ MSCloudInfo *getDefaultCloudInfo() { return getCloudInfo(dfltCloudRefId); }
4773+
4774+ uint32_t newCloudInfo(uint32_t id, const char *server, const char *bucket, const char *publicKey, const char *privateKey, bool isDefault );
4775+
4776+
4777+ void deleteCloudInfo(uint32_t id);
4778+ void setDefaultCloudRef(uint32_t dflt);
4779+ uint32_t getDefaultCloudRef() { return dfltCloudRefId;}
4780+
4781+ void getNewKey(CloudKeyPtr key);
4782+ CSString *getObjectKey(CloudKeyPtr key);
4783+ void setKeepAlive(uint32_t keep_alive_arg) {keep_alive = keep_alive_arg;}
4784+
4785+ void createDB();
4786+ void dropDB();
4787+ bool dbExists();
4788+
4789+ void putData( CloudKeyPtr key, CSInputStream *stream, off64_t size);
4790+ off64_t getData(CloudKeyPtr key, char *data, off64_t size);
4791+ CSString *getDataURL(CloudKeyPtr key);
4792+ void deleteData(CloudKeyPtr key);
4793+ CSString *getSignature(CloudKeyPtr key, CSString *content_type, uint32_t *s3AuthorizationTime);
4794
4795 };
4796
4797
4798=== removed file 'plugin/pbms/src/compactor_ms.cc'
4799--- plugin/pbms/src/compactor_ms.cc 2010-12-18 04:43:40 +0000
4800+++ plugin/pbms/src/compactor_ms.cc 1970-01-01 00:00:00 +0000
4801@@ -1,343 +0,0 @@
4802-/* Copyright (C) 2008 PrimeBase Technologies GmbH, Germany
4803- *
4804- * PrimeBase Media Stream for MySQL
4805- *
4806- * This program is free software; you can redistribute it and/or modify
4807- * it under the terms of the GNU General Public License as published by
4808- * the Free Software Foundation; either version 2 of the License, or
4809- * (at your option) any later version.
4810- *
4811- * This program is distributed in the hope that it will be useful,
4812- * but WITHOUT ANY WARRANTY; without even the implied warranty of
4813- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
4814- * GNU General Public License for more details.
4815- *
4816- * You should have received a copy of the GNU General Public License
4817- * along with this program; if not, write to the Free Software
4818- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
4819- *
4820- * Original author: Paul McCullagh
4821- * Continued development: Barry Leslie
4822- *
4823- * 2007-07-10
4824- *
4825- * H&G2JCtL
4826- *
4827- * Network interface.
4828- *
4829- */
4830-
4831-#include "cslib/CSConfig.h"
4832-
4833-#include "defs_ms.h"
4834-
4835-#include "cslib/CSGlobal.h"
4836-#include "cslib/CSStrUtil.h"
4837-#include "cslib/CSStorage.h"
4838-
4839-#include "compactor_ms.h"
4840-#include "open_table_ms.h"
4841-#include "repository_ms.h"
4842-#include "parameters_ms.h"
4843-
4844-/*
4845- * ---------------------------------------------------------------
4846- * COMPACTOR THREAD
4847- */
4848-
4849-MSCompactorThread::MSCompactorThread(time_t wait_time, MSDatabase *db):
4850-CSDaemon(wait_time, NULL),
4851-iCompactorDatabase(db)
4852-{
4853-}
4854-
4855-void MSCompactorThread::close()
4856-{
4857-}
4858-
4859-bool MSCompactorThread::doWork()
4860-{
4861- bool complete;
4862- MSRepository *src_repo, *dst_repo;
4863- MSRepoFile *src_file, *dst_file;
4864- uint32_t src_repo_id;
4865- MSBlobHeadRec blob;
4866- off64_t src_offset;
4867- uint16_t head_size;
4868- uint64_t blob_size, blob_data_size;
4869- CSStringBuffer *head;
4870- MSRepoPointersRec ptr;
4871- uint32_t table_ref_count;
4872- uint32_t blob_ref_count;
4873- int ref_count;
4874- size_t ref_size;
4875- CSMutex *mylock;
4876- uint32_t tab_id;
4877- uint64_t blob_id;
4878- MSOpenTable *otab;
4879- uint32_t repo_id;
4880- uint64_t repo_offset;
4881- uint64_t repo_blob_size;
4882- uint16_t repo_head_size;
4883- uint16_t tab_index;
4884- uint8_t status;
4885-
4886- enter_();
4887- retry:
4888-
4889-#ifdef MS_COMPACTOR_POLLS
4890- if (!(src_repo = iCompactorDatabase->getRepoFullOfTrash(NULL)))
4891- return_(true);
4892-#else
4893- myWaitTime = MS_DEFAULT_COMPACTOR_WAIT * 1000; // Time in milli-seconds
4894- if (!(src_repo = iCompactorDatabase->getRepoFullOfTrash(&myWaitTime)))
4895- return_(true);
4896-#endif
4897- frompool_(src_repo);
4898- src_file = src_repo->openRepoFile();
4899- push_(src_file);
4900-
4901- dst_repo = iCompactorDatabase->lockRepo(src_repo->myRepoFileSize - src_repo->myGarbageCount);
4902- frompool_(dst_repo);
4903- dst_file = dst_repo->openRepoFile();
4904- push_(dst_file);
4905-
4906- new_(head, CSStringBuffer(100));
4907- push_(head);
4908-
4909- complete = false;
4910- src_repo_id = src_repo->myRepoID;
4911- src_offset = src_repo->myRepoHeadSize;
4912- //printf("\nCompacting repo %"PRId32"\n\n", src_repo_id);
4913- // For testing:
4914- {
4915- int blockit = 0;
4916- if (blockit) {
4917- release_(head);
4918- release_(dst_file);
4919- backtopool_(dst_repo);
4920- release_(src_file);
4921- backtopool_(src_repo);
4922-
4923- myWaitTime = 5 * 1000; // Time in milli-seconds
4924- return_(true);
4925- }
4926- }
4927- while (src_offset < src_repo->myRepoFileSize) {
4928- retry_loop:
4929- suspended();
4930-
4931- if (myMustQuit)
4932- goto quit;
4933- retry_read:
4934-
4935- // A lock is required here because references and dereferences to the
4936- // BLOBs can result in the repository record being updated while
4937- // it is being copied.
4938- mylock = &src_repo->myRepoLock[src_offset % CS_REPO_REC_LOCK_COUNT];
4939- lock_(mylock);
4940- if (src_file->read(&blob, src_offset, src_repo->myRepoBlobHeadSize, 0) < src_repo->myRepoBlobHeadSize) {
4941- unlock_(mylock);
4942- break;
4943- }
4944- ref_size = CS_GET_DISK_1(blob.rb_ref_size_1);
4945- ref_count = CS_GET_DISK_2(blob.rb_ref_count_2);
4946- head_size = CS_GET_DISK_2(blob.rb_head_size_2);
4947- blob_size = CS_GET_DISK_6(blob.rb_blob_repo_size_6);
4948- blob_data_size = CS_GET_DISK_6(blob.rb_blob_data_size_6);
4949- status = CS_GET_DISK_1(blob.rb_status_1);
4950- if ((blob_data_size == 0) || ref_count <= 0 || ref_size == 0 ||
4951- head_size < src_repo->myRepoBlobHeadSize + ref_count * ref_size ||
4952- !VALID_BLOB_STATUS(status)) {
4953- /* Can't be true. Assume this is garbage! */
4954- unlock_(mylock);
4955- src_offset++;
4956- goto retry_read;
4957- }
4958- if (IN_USE_BLOB_STATUS(status)) {
4959- head->setLength(head_size);
4960- if (src_file->read(head->getBuffer(0), src_offset, head_size, 0) != head_size) {
4961- unlock_(mylock);
4962- break;
4963- }
4964-
4965- table_ref_count = 0;
4966- blob_ref_count = 0;
4967-
4968- ptr.rp_chars = head->getBuffer(0) + src_repo->myRepoBlobHeadSize;
4969- for (int count = 0; count < ref_count; count++) {
4970- switch (CS_GET_DISK_2(ptr.rp_ref->rr_type_2)) {
4971- case MS_BLOB_FREE_REF:
4972- break;
4973- case MS_BLOB_TABLE_REF:
4974- /* Check the reference: */
4975- tab_id = CS_GET_DISK_4(ptr.rp_tab_ref->tr_table_id_4);
4976- blob_id = CS_GET_DISK_6(ptr.rp_tab_ref->tr_blob_id_6);
4977-
4978- otab = MSTableList::getOpenTableByID(iCompactorDatabase->myDatabaseID, tab_id);
4979- if (otab) {
4980- frompool_(otab);
4981- /* Ignore the return value (it will fail because auth_code is wrong!)!! */
4982- uint32_t auth_code = 0;
4983- otab->getDBTable()->readBlobHandle(otab, blob_id, &auth_code, &repo_id, &repo_offset, &repo_blob_size, &repo_head_size, false);
4984- backtopool_(otab);
4985- if (repo_id == src_repo_id &&
4986- repo_offset == src_offset &&
4987- repo_blob_size == blob_data_size &&
4988- repo_head_size == head_size)
4989- table_ref_count++;
4990- else
4991- /* Remove the reference: */
4992- CS_SET_DISK_2(ptr.rp_ref->rr_type_2, MS_BLOB_FREE_REF);
4993- }
4994- else
4995- CS_SET_DISK_2(ptr.rp_ref->rr_type_2, MS_BLOB_FREE_REF);
4996- break;
4997- case MS_BLOB_DELETE_REF:
4998- /* These are temporary references from the TempLog file. */
4999- /* We try to prevent this from happening, but it can! */
5000- uint32_t temp_log_id;
The diff has been truncated for viewing.