1/*
2 * Copyright (C) by Klaas Freitag <freitag@owncloud.com>
3 *
4 * This library is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Lesser General Public
6 * License as published by the Free Software Foundation; either
7 * version 2.1 of the License, or (at your option) any later version.
8 *
9 * This library is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Lesser General Public License for more details.
13 *
14 * You should have received a copy of the GNU Lesser General Public
15 * License along with this library; if not, write to the Free Software
16 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 */
18
19#ifndef SYNCJOURNALDB_H
20#define SYNCJOURNALDB_H
21
22#include <QObject>
23#include <qmutex.h>
24#include <QDateTime>
25#include <QHash>
26#include <functional>
27
28#include "common/utility.h"
29#include "common/ownsql.h"
30#include "common/syncjournalfilerecord.h"
31
32namespace OCC {
33class SyncJournalFileRecord;
34
35/**
36 * @brief Class that handles the sync database
37 *
38 * This class is thread safe. All public functions lock the mutex.
39 * @ingroup libsync
40 */
41class OCSYNC_EXPORT SyncJournalDb : public QObject
42{
43 Q_OBJECT
44public:
45 explicit SyncJournalDb(const QString &dbFilePath, QObject *parent = 0);
46 virtual ~SyncJournalDb();
47
48 /// Create a journal path for a specific configuration
49 static QString makeDbName(const QString &localPath,
50 const QUrl &remoteUrl,
51 const QString &remotePath,
52 const QString &user);
53
54 /// Migrate a csync_journal to the new path, if necessary. Returns false on error
55 static bool maybeMigrateDb(const QString &localPath, const QString &absoluteJournalPath);
56
57 // To verify that the record could be found check with SyncJournalFileRecord::isValid()
58 bool getFileRecord(const QString &filename, SyncJournalFileRecord *rec) { return getFileRecord(filename.toUtf8(), rec); }
59 bool getFileRecord(const QByteArray &filename, SyncJournalFileRecord *rec);
60 bool getFileRecordByInode(quint64 inode, SyncJournalFileRecord *rec);
61 bool getFileRecordsByFileId(const QByteArray &fileId, const std::function<void(const SyncJournalFileRecord &)> &rowCallback);
62 bool getFilesBelowPath(const QByteArray &path, const std::function<void(const SyncJournalFileRecord&)> &rowCallback);
63 bool setFileRecord(const SyncJournalFileRecord &record);
64
65 /// Like setFileRecord, but preserves checksums
66 bool setFileRecordMetadata(const SyncJournalFileRecord &record);
67
68 bool deleteFileRecord(const QString &filename, bool recursively = false);
69 bool updateFileRecordChecksum(const QString &filename,
70 const QByteArray &contentChecksum,
71 const QByteArray &contentChecksumType);
72 bool updateLocalMetadata(const QString &filename,
73 qint64 modtime, quint64 size, quint64 inode);
74 bool exists();
75 void walCheckpoint();
76
77 QString databaseFilePath() const;
78
79 static qint64 getPHash(const QByteArray &);
80
81 void setErrorBlacklistEntry(const SyncJournalErrorBlacklistRecord &item);
82 void wipeErrorBlacklistEntry(const QString &file);
83 void wipeErrorBlacklistCategory(SyncJournalErrorBlacklistRecord::Category category);
84 int wipeErrorBlacklist();
85 int errorBlackListEntryCount();
86
87 struct DownloadInfo
88 {
89 DownloadInfo()
90 : _errorCount(0)
91 , _valid(false)
92 {
93 }
94 QString _tmpfile;
95 QByteArray _etag;
96 int _errorCount;
97 bool _valid;
98 };
99 struct UploadInfo
100 {
101 UploadInfo()
102 : _chunk(0)
103 , _transferid(0)
104 , _size(0)
105 , _errorCount(0)
106 , _valid(false)
107 {
108 }
109 int _chunk;
110 int _transferid;
111 qint64 _size;
112 qint64 _modtime;
113 int _errorCount;
114 bool _valid;
115 QByteArray _contentChecksum;
116 /**
117 * Returns true if this entry refers to a chunked upload that can be continued.
118 * (As opposed to a small file transfer which is stored in the db so we can detect the case
119 * when the upload succeeded, but the connection was dropped before we got the answer)
120 */
121 bool isChunked() const { return _transferid != 0; }
122 };
123
124 struct PollInfo
125 {
126 QString _file;
127 QString _url;
128 qint64 _modtime;
129 };
130
131 DownloadInfo getDownloadInfo(const QString &file);
132 void setDownloadInfo(const QString &file, const DownloadInfo &i);
133 QVector<DownloadInfo> getAndDeleteStaleDownloadInfos(const QSet<QString> &keep);
134 int downloadInfoCount();
135
136 UploadInfo getUploadInfo(const QString &file);
137 void setUploadInfo(const QString &file, const UploadInfo &i);
138 // Return the list of transfer ids that were removed.
139 QVector<uint> deleteStaleUploadInfos(const QSet<QString> &keep);
140
141 SyncJournalErrorBlacklistRecord errorBlacklistEntry(const QString &);
142 bool deleteStaleErrorBlacklistEntries(const QSet<QString> &keep);
143
144 void avoidRenamesOnNextSync(const QString &path) { avoidRenamesOnNextSync(path.toUtf8()); }
145 void avoidRenamesOnNextSync(const QByteArray &path);
146 void setPollInfo(const PollInfo &);
147 QVector<PollInfo> getPollInfos();
148
149 enum SelectiveSyncListType {
150 /** The black list is the list of folders that are unselected in the selective sync dialog.
151 * For the sync engine, those folders are considered as if they were not there, so the local
152 * folders will be deleted */
153 SelectiveSyncBlackList = 1,
154 /** When a shared folder has a size bigger than a configured size, it is by default not sync'ed
155 * Unless it is in the white list, in which case the folder is sync'ed and all its children.
156 * If a folder is both on the black and the white list, the black list wins */
157 SelectiveSyncWhiteList = 2,
158 /** List of big sync folders that have not been confirmed by the user yet and that the UI
159 * should notify about */
160 SelectiveSyncUndecidedList = 3
161 };
162 /* return the specified list from the database */
163 QStringList getSelectiveSyncList(SelectiveSyncListType type, bool *ok);
164 /* Write the selective sync list (remove all other entries of that list */
165 void setSelectiveSyncList(SelectiveSyncListType type, const QStringList &list);
166
167 /**
168 * Make sure that on the next sync fileName and its parents are discovered from the server.
169 *
170 * That means its metadata and, if it's a directory, its direct contents.
171 *
172 * Specifically, etag (md5 field) of fileName and all its parents are set to _invalid_.
173 * That causes a metadata difference and a resulting discovery from the remote for the
174 * affected folders.
175 *
176 * Since folders in the selective sync list will not be rediscovered (csync_ftw,
177 * _csync_detect_update skip them), the _invalid_ marker will stay. And any
178 * child items in the db will be ignored when reading a remote tree from the database.
179 *
180 * Any setFileRecord() call to affected directories before the next sync run will be
181 * adjusted to retain the invalid etag via _etagStorageFilter.
182 */
183 void avoidReadFromDbOnNextSync(const QString &fileName) { avoidReadFromDbOnNextSync(fileName.toUtf8()); }
184 void avoidReadFromDbOnNextSync(const QByteArray &fileName);
185
186 /**
187 * Wipe _etagStorageFilter. Also done implicitly on close().
188 */
189 void clearEtagStorageFilter();
190
191 /**
192 * Ensures full remote discovery happens on the next sync.
193 *
194 * Equivalent to calling avoidReadFromDbOnNextSync() for all files.
195 */
196 void forceRemoteDiscoveryNextSync();
197
198 bool postSyncCleanup(const QSet<QString> &filepathsToKeep,
199 const QSet<QString> &prefixesToKeep);
200
201 /* Because sqlite transactions are really slow, we encapsulate everything in big transactions
202 * Commit will actually commit the transaction and create a new one.
203 */
204 void commit(const QString &context, bool startTrans = true);
205 void commitIfNeededAndStartNewTransaction(const QString &context);
206
207 void close();
208
209 /**
210 * return true if everything is correct
211 */
212 bool isConnected();
213
214 /**
215 * Returns the checksum type for an id.
216 */
217 QByteArray getChecksumType(int checksumTypeId);
218
219 /**
220 * The data-fingerprint used to detect backup
221 */
222 void setDataFingerprint(const QByteArray &dataFingerprint);
223 QByteArray dataFingerprint();
224
225
226 // Conflict record functions
227
228 /// Store a new or updated record in the database
229 void setConflictRecord(const ConflictRecord &record);
230
231 /// Retrieve a conflict record by path of the file with the conflict tag
232 ConflictRecord conflictRecord(const QByteArray &path);
233
234 /// Delete a conflict record by path of the file with the conflict tag
235 void deleteConflictRecord(const QByteArray &path);
236
237 /// Return all paths of files with a conflict tag in the name and records in the db
238 QByteArrayList conflictRecordPaths();
239
240
241 /**
242 * Delete any file entry. This will force the next sync to re-sync everything as if it was new,
243 * restoring everyfile on every remote. If a file is there both on the client and server side,
244 * it will be a conflict that will be automatically resolved if the file is the same.
245 */
246 void clearFileTable();
247
248 /**
249 * Set the 'ItemTypeVirtualFileDownload' to all the files that have the ItemTypeVirtualFile flag
250 * within the directory specified path path
251 */
252 void markVirtualFileForDownloadRecursively(const QByteArray &path);
253
254private:
255 int getFileRecordCount();
256 bool updateDatabaseStructure();
257 bool updateMetadataTableStructure();
258 bool updateErrorBlacklistTableStructure();
259 bool sqlFail(const QString &log, const SqlQuery &query);
260 void commitInternal(const QString &context, bool startTrans = true);
261 void startTransaction();
262 void commitTransaction();
263 QVector<QByteArray> tableColumns(const QByteArray &table);
264 bool checkConnect();
265
266 // Same as forceRemoteDiscoveryNextSync but without acquiring the lock
267 void forceRemoteDiscoveryNextSyncLocked();
268
269 // Returns the integer id of the checksum type
270 //
271 // Returns 0 on failure and for empty checksum types.
272 int mapChecksumType(const QByteArray &checksumType);
273
274 SqlDatabase _db;
275 QString _dbFile;
276 QMutex _mutex; // Public functions are protected with the mutex.
277 int _transaction;
278 bool _metadataTableIsEmpty;
279
280 SqlQuery _getFileRecordQuery;
281 SqlQuery _getFileRecordQueryByInode;
282 SqlQuery _getFileRecordQueryByFileId;
283 SqlQuery _getFilesBelowPathQuery;
284 SqlQuery _getAllFilesQuery;
285 SqlQuery _setFileRecordQuery;
286 SqlQuery _setFileRecordChecksumQuery;
287 SqlQuery _setFileRecordLocalMetadataQuery;
288 SqlQuery _getDownloadInfoQuery;
289 SqlQuery _setDownloadInfoQuery;
290 SqlQuery _deleteDownloadInfoQuery;
291 SqlQuery _getUploadInfoQuery;
292 SqlQuery _setUploadInfoQuery;
293 SqlQuery _deleteUploadInfoQuery;
294 SqlQuery _deleteFileRecordPhash;
295 SqlQuery _deleteFileRecordRecursively;
296 SqlQuery _getErrorBlacklistQuery;
297 SqlQuery _setErrorBlacklistQuery;
298 SqlQuery _getSelectiveSyncListQuery;
299 SqlQuery _getChecksumTypeIdQuery;
300 SqlQuery _getChecksumTypeQuery;
301 SqlQuery _insertChecksumTypeQuery;
302 SqlQuery _getDataFingerprintQuery;
303 SqlQuery _setDataFingerprintQuery1;
304 SqlQuery _setDataFingerprintQuery2;
305 SqlQuery _getConflictRecordQuery;
306 SqlQuery _setConflictRecordQuery;
307 SqlQuery _deleteConflictRecordQuery;
308
309 /* Storing etags to these folders, or their parent folders, is filtered out.
310 *
311 * When avoidReadFromDbOnNextSync() is called some etags to _invalid_ in the
312 * database. If this is done during a sync run, a later propagation job might
313 * undo that by writing the correct etag to the database instead. This filter
314 * will prevent this write and instead guarantee the _invalid_ etag stays in
315 * place.
316 *
317 * The list is cleared on close() (end of sync run) and explicitly with
318 * clearEtagStorageFilter() (start of sync run).
319 *
320 * The contained paths have a trailing /.
321 */
322 QList<QByteArray> _etagStorageFilter;
323
324 /** The journal mode to use for the db.
325 *
326 * Typically WAL initially, but may be set to other modes via environment
327 * variable, for specific filesystems, or when WAL fails in a particular way.
328 */
329 QByteArray _journalMode;
330};
331
332bool OCSYNC_EXPORT
333operator==(const SyncJournalDb::DownloadInfo &lhs,
334 const SyncJournalDb::DownloadInfo &rhs);
335bool OCSYNC_EXPORT
336operator==(const SyncJournalDb::UploadInfo &lhs,
337 const SyncJournalDb::UploadInfo &rhs);
338
339} // namespace OCC
340#endif // SYNCJOURNALDB_H
341