1/*
2 * Copyright (C) 2003-2006 Ben van Klinken and the CLucene Team
3 *
4 * Distributable under the terms of either the Apache License (Version 2.0) or
5 * the GNU Lesser General Public License, as specified in the COPYING file.
6 *
7 * Changes are Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies).
8*/
9#include <QtCore/QDir>
10#include <QtCore/QDateTime>
11#include <QtCore/QFileInfo>
12#include <QtCore/QByteArray>
13#include <QtCore/QCryptographicHash>
14
15#include "CLucene/StdHeader.h"
16#include "FSDirectory.h"
17#include "CLucene/index/IndexReader.h"
18#include "CLucene/util/Misc.h"
19#include "CLucene/debug/condition.h"
20
21CL_NS_DEF(store)
22CL_NS_USE(util)
23
24bool FSDirectory::disableLocks = false;
25
26// This cache of directories ensures that there is a unique Directory instance
27// per path, so that synchronization on the Directory can be used to synchronize
28// access between readers and writers.
29static CL_NS(util)::CLHashMap<QString, FSDirectory*,
30 CL_NS(util)::Compare::Qstring, CL_NS(util)::Equals::Qstring,
31 CL_NS(util)::Deletor::DummyQString> DIRECTORIES(false, false);
32
33// # pragma mark -- FSDirectory::FSLock
34
35FSDirectory::FSLock::FSLock(const QString& _lockDir, const QString& name)
36 : lockDir(_lockDir)
37 , lockFile(_lockDir + QDir::separator() + name)
38{
39}
40
41FSDirectory::FSLock::~FSLock()
42{
43}
44
45bool FSDirectory::FSLock::obtain()
46{
47 if (disableLocks)
48 return true;
49
50 if (QFile::exists(lockFile))
51 return false;
52
53 QDir dir(lockDir);
54 if (!dir.exists()) {
55 if (!dir.mkpath(lockDir)) {
56 // 34: len of "Couldn't create lock directory: "
57 char* err = _CL_NEWARRAY(
58 char, 34 + strlen(lockDir.toLocal8Bit().constData()) + 1);
59 strcpy(err, "Couldn't create lock directory: ");
60 strcat(err, lockDir.toLocal8Bit().constData());
61 _CLTHROWA_DEL(CL_ERR_IO, err);
62 }
63 }
64
65 QFile file(lockFile);
66 return file.open(QIODevice::ReadWrite);
67}
68
69void FSDirectory::FSLock::release()
70{
71 if (disableLocks)
72 return;
73
74 QFile file(lockFile);
75 file.remove();
76}
77
78bool FSDirectory::FSLock::isLocked()
79{
80 if (disableLocks)
81 return false;
82 return QFile::exists(lockFile);
83}
84
85QString FSDirectory::FSLock::toString() const
86{
87 QString ret(QLatin1String("Lock@"));
88 return ret.append(lockFile);
89}
90
91// # pragma mark -- FSDirectory::FSIndexInput
92
93FSDirectory::FSIndexInput::FSIndexInput(const QString& path, int32_t bufferSize)
94 : BufferedIndexInput(bufferSize)
95{
96 CND_PRECONDITION(!path.isEmpty(), "path is NULL");
97
98 handle = _CLNEW SharedHandle();
99 handle->fhandle.setFileName(path);
100 handle->fhandle.open(QIODevice::ReadOnly);
101
102 if (handle->fhandle.error() != QFile::NoError) {
103 switch(handle->fhandle.error()) {
104 case 1:
105 _CLTHROWA(CL_ERR_IO, "An error occurred when reading from the file");
106 break;
107 case 2:
108 _CLTHROWA(CL_ERR_IO, "An error occurred when writing to the file.");
109 break;
110 case 5:
111 _CLTHROWA(CL_ERR_IO, "The file could not be opened.");
112 break;
113 case 6:
114 _CLTHROWA(CL_ERR_IO, "The operation was aborted.");
115 break;
116 case 7:
117 _CLTHROWA(CL_ERR_IO, "A timeout occurred.");
118 break;
119 case 8:
120 _CLTHROWA(CL_ERR_IO, "An unspecified error occurred.");
121 break;
122 case 9:
123 _CLTHROWA(CL_ERR_IO, "The file could not be removed.");
124 break;
125 case 10:
126 _CLTHROWA(CL_ERR_IO, "The file could not be renamed.");
127 break;
128 case 11:
129 _CLTHROWA(CL_ERR_IO, "The position in the file could not be changed.");
130 break;
131 case 12:
132 _CLTHROWA(CL_ERR_IO, "The file could not be resized.e");
133 break;
134 case 13:
135 _CLTHROWA(CL_ERR_IO, "The file could not be accessed.");
136 break;
137 case 14:
138 _CLTHROWA(CL_ERR_IO, "The file could not be copied.");
139 break;
140 case 4:
141 default:
142 _CLTHROWA(CL_ERR_IO, "A fatal error occurred.");
143 }
144 }
145
146 //Store the file length
147 handle->_length = handle->fhandle.size();
148 handle->_fpos = 0;
149 this->_pos = 0;
150}
151
152FSDirectory::FSIndexInput::FSIndexInput(const FSIndexInput& other)
153 : BufferedIndexInput(other)
154{
155 if (other.handle == NULL)
156 _CLTHROWA(CL_ERR_NullPointer, "other handle is null");
157
158 SCOPED_LOCK_MUTEX(*other.handle->THIS_LOCK)
159
160 _pos = other.handle->_fpos;
161 handle = _CL_POINTER(other.handle);
162}
163
164FSDirectory::FSIndexInput::~FSIndexInput()
165{
166 FSIndexInput::close();
167}
168
169void FSDirectory::FSIndexInput::close()
170{
171 BufferedIndexInput::close();
172#ifdef _LUCENE_THREADMUTEX
173 if (handle != NULL) {
174 // Here we have a bit of a problem... We need to lock the handle to
175 // ensure that we can safely delete the handle... But if we delete the
176 // handle, then the scoped unlock, won't be able to unlock the mutex...
177
178 // take a reference of the lock object...
179 _LUCENE_THREADMUTEX* mutex = handle->THIS_LOCK;
180 //lock the mutex
181 mutex->lock();
182
183 // determine if we are about to delete the handle...
184 bool doUnlock = (handle->__cl_refcount > 1);
185 // decdelete (deletes if refcount is down to 0)
186 _CLDECDELETE(handle);
187
188 if (doUnlock)
189 mutex->unlock();
190 else
191 delete mutex;
192 }
193#else
194 _CLDECDELETE(handle);
195#endif
196}
197
198IndexInput* FSDirectory::FSIndexInput::clone() const
199{
200 return _CLNEW FSDirectory::FSIndexInput(*this);
201}
202
203void FSDirectory::FSIndexInput::seekInternal(const int64_t position)
204{
205 CND_PRECONDITION(position >= 0 && position < handle->_length,
206 "Seeking out of range")
207 _pos = position;
208}
209
210void FSDirectory::FSIndexInput::readInternal(uint8_t* b, const int32_t len)
211{
212 SCOPED_LOCK_MUTEX(*handle->THIS_LOCK)
213
214 CND_PRECONDITION(handle != NULL, "shared file handle has closed");
215 CND_PRECONDITION(handle->fhandle.isOpen(), "file is not open");
216
217 if (handle->_fpos != _pos) {
218 handle->fhandle.seek(_pos);
219 if (handle->fhandle.pos() != _pos)
220 _CLTHROWA( CL_ERR_IO, "File IO Seek error");
221 handle->_fpos = _pos;
222 }
223
224 bufferLength = (int32_t)handle->fhandle.read((char*)b, len);
225 if (bufferLength == 0)
226 _CLTHROWA(CL_ERR_IO, "read past EOF");
227
228 if (bufferLength == -1)
229 _CLTHROWA(CL_ERR_IO, "read error");
230
231 _pos += bufferLength;
232 handle->_fpos =_pos;
233}
234
235// # pragma mark -- FSDirectory::FSIndexInput::SharedHandle
236
237FSDirectory::FSIndexInput::SharedHandle::SharedHandle()
238 : _fpos(0)
239 , _length(0)
240{
241#ifdef _LUCENE_THREADMUTEX
242 THIS_LOCK = new _LUCENE_THREADMUTEX;
243#endif
244}
245
246FSDirectory::FSIndexInput::SharedHandle::~SharedHandle()
247{
248 if (fhandle.isOpen())
249 fhandle.close();
250}
251
252// # pragma mark -- FSDirectory::FSIndexOutput
253
254FSDirectory::FSIndexOutput::FSIndexOutput(const QString& path)
255{
256 //O_BINARY - Opens file in binary (untranslated) mode
257 //O_CREAT - Creates and opens new file for writing. Has no effect if file specified by filename exists
258 //O_RANDOM - Specifies that caching is optimized for, but not restricted to, random access from disk.
259 //O_WRONLY - Opens file for writing only;
260 fhandle.setFileName(path);
261 fhandle.open(QIODevice::ReadWrite | QIODevice::Truncate);
262
263 if (fhandle.error() != QFile::NoError) {
264 switch(fhandle.error()) {
265 case 1:
266 _CLTHROWA(CL_ERR_IO, "An error occurred when reading from the file");
267 break;
268 case 2:
269 _CLTHROWA(CL_ERR_IO, "An error occurred when writing to the file.");
270 break;
271 case 5:
272 _CLTHROWA(CL_ERR_IO, "The file could not be opened.");
273 break;
274 case 6:
275 _CLTHROWA(CL_ERR_IO, "The operation was aborted.");
276 break;
277 case 7:
278 _CLTHROWA(CL_ERR_IO, "A timeout occurred.");
279 break;
280 case 8:
281 _CLTHROWA(CL_ERR_IO, "An unspecified error occurred.");
282 break;
283 case 9:
284 _CLTHROWA(CL_ERR_IO, "The file could not be removed.");
285 break;
286 case 10:
287 _CLTHROWA(CL_ERR_IO, "The file could not be renamed.");
288 break;
289 case 11:
290 _CLTHROWA(CL_ERR_IO, "The position in the file could not be changed.");
291 break;
292 case 12:
293 _CLTHROWA(CL_ERR_IO, "The file could not be resized.e");
294 break;
295 case 13:
296 _CLTHROWA(CL_ERR_IO, "The file could not be accessed.");
297 break;
298 case 14:
299 _CLTHROWA(CL_ERR_IO, "The file could not be copied.");
300 break;
301 case 4:
302 default:
303 _CLTHROWA(CL_ERR_IO, "A fatal error occurred.");
304 }
305 }
306}
307
308FSDirectory::FSIndexOutput::~FSIndexOutput()
309{
310 if (fhandle.isOpen()) {
311 try {
312 FSIndexOutput::close();
313 } catch (CLuceneError& err) {
314 //ignore IO errors...
315 if (err.number() != CL_ERR_IO)
316 throw;
317 }
318 }
319}
320
321void FSDirectory::FSIndexOutput::close()
322{
323 try {
324 BufferedIndexOutput::close();
325 } catch (CLuceneError& err) {
326 //ignore IO errors...
327 if (err.number() != CL_ERR_IO)
328 throw;
329 }
330 fhandle.close();
331}
332
333int64_t FSDirectory::FSIndexOutput::length()
334{
335 CND_PRECONDITION(fhandle.isOpen(), "file is not open");
336 return fhandle.size();
337}
338
339void FSDirectory::FSIndexOutput::seek(const int64_t pos)
340{
341 CND_PRECONDITION(fhandle.isOpen(), "file is not open");
342
343 BufferedIndexOutput::seek(pos);
344 fhandle.seek(pos);
345 if (fhandle.pos() != pos)
346 _CLTHROWA(CL_ERR_IO, "File IO Seek error");
347}
348
349void FSDirectory::FSIndexOutput::flushBuffer(const uint8_t* b, const int32_t size)
350{
351 CND_PRECONDITION(fhandle.isOpen(), "file is not open");
352
353 if (size > 0 && fhandle.write((const char*)b, size) != size)
354 _CLTHROWA(CL_ERR_IO, "File IO Write error");
355}
356
357// # pragma mark -- FSDirectory
358
359FSDirectory::FSDirectory(const QString& path, const bool createDir)
360 : Directory()
361 , refCount(0)
362 , useMMap(false)
363{
364 //set a realpath so that if we change directory, we can still function
365 directory = QFileInfo(path).absoluteFilePath();
366 lockDir = directory;
367
368 QDir dir(lockDir);
369 if (!dir.exists()) {
370 if (!dir.mkpath(lockDir))
371 _CLTHROWA_DEL(CL_ERR_IO, "Cannot create temp directory");
372 }
373
374 QFileInfo info(lockDir);
375 if (info.isFile() || info.isSymLink())
376 _CLTHROWA(CL_ERR_IO, "Found regular file where directory expected");
377
378 if (createDir)
379 create();
380
381 dir.setPath(directory);
382 if (!dir.exists()) {
383 //19: len of " is not a directory"
384 char* err =
385 _CL_NEWARRAY(char, 19 + strlen(path.toLocal8Bit().constData()) + 1);
386 strcpy(err, path.toLocal8Bit().constData());
387 strcat(err, " is not a directory");
388 _CLTHROWA_DEL(CL_ERR_IO, err);
389 }
390}
391
392void FSDirectory::create()
393{
394 SCOPED_LOCK_MUTEX(THIS_LOCK)
395
396 bool clear = false;
397 QDir dir(directory);
398 if (!dir.exists()) {
399 if (!dir.mkpath(directory)) {
400 char* err = _CL_NEWARRAY( // 27 len of "Couldn't create directory:"
401 char, 27 + strlen(directory.toLocal8Bit().constData()) + 1);
402 strcpy(err, "Couldn't create directory: ");
403 strcat(err, directory.toLocal8Bit().constData());
404 _CLTHROWA_DEL(CL_ERR_IO, err);
405 }
406 } else {
407 clear = true;
408 }
409
410 QFileInfo info(directory);
411 if (info.isFile() || info.isSymLink()) {
412 char tmp[1024];
413 _snprintf(tmp, 1024, "%s not a directory",
414 directory.toLocal8Bit().constData());
415 _CLTHROWA(CL_ERR_IO, tmp);
416 }
417
418 if (clear) {
419 dir.setPath(directory);
420 // clear probably existing lucene index files
421 QStringList fileList = dir.entryList(QDir::Files | QDir::Hidden
422 | QDir::NoSymLinks);
423 foreach(const QString file, fileList) {
424 if (CL_NS(index)::IndexReader::isLuceneFile(file)) {
425 if (!dir.remove(file))
426 _CLTHROWA(CL_ERR_IO, "Couldn't delete file ");
427 }
428 }
429
430 // clear probably existing file locks
431 QFileInfo dirInfo(lockDir);
432 if (dirInfo.exists() && dirInfo.isReadable() && dirInfo.isWritable()
433 && !dirInfo.isFile() && !dirInfo.isSymLink()) {
434 QDir lockDirectory(lockDir);
435 fileList = dir.entryList(QStringList() << getLockPrefix()
436 + QLatin1Char('*'), QDir::Files | QDir::Hidden | QDir::NoSymLinks);
437
438 foreach(const QString file, fileList) {
439 if (!lockDirectory.remove(file))
440 _CLTHROWA(CL_ERR_IO, "Couldn't delete file ");
441 }
442 }
443 else {
444 //todo: richer error: + lockDir.getAbsolutePath());
445 _CLTHROWA(CL_ERR_IO, "Cannot read lock directory");
446 }
447 }
448}
449
450void FSDirectory::priv_getFN(QString& buffer, const QString& name) const
451{
452 buffer.clear();
453 buffer.append(directory);
454 buffer.append(QDir::separator());
455 buffer.append(name);
456}
457
458FSDirectory::~FSDirectory()
459{
460}
461
462QStringList FSDirectory::list() const
463{
464 CND_PRECONDITION(!directory.isEmpty(), "directory is not open");
465
466 QDir dir(directory);
467 return dir.entryList(QDir::Files | QDir::Hidden);
468}
469
470bool FSDirectory::fileExists(const QString& name) const
471{
472 CND_PRECONDITION(!directory.isEmpty(), "directory is not open");
473
474 QDir dir(directory);
475 return dir.entryList().contains(name);
476}
477
478QString FSDirectory::getDirName() const
479{
480 return directory;
481}
482
483//static
484FSDirectory* FSDirectory::getDirectory(const QString& file, const bool _create)
485{
486 FSDirectory* dir = NULL;
487 {
488 if (file.isEmpty())
489 _CLTHROWA(CL_ERR_IO, "Invalid directory");
490
491 SCOPED_LOCK_MUTEX(DIRECTORIES.THIS_LOCK)
492 dir = DIRECTORIES.get(file);
493 if ( dir == NULL ){
494 dir = _CLNEW FSDirectory(file, _create);
495 DIRECTORIES.put(dir->directory, dir);
496 } else if (_create) {
497 dir->create();
498 }
499
500 {
501 SCOPED_LOCK_MUTEX(dir->THIS_LOCK)
502 dir->refCount++;
503 }
504 }
505
506 return _CL_POINTER(dir);
507}
508
509int64_t FSDirectory::fileModified(const QString& name) const
510{
511 CND_PRECONDITION(!directory.isEmpty(), "directory is not open");
512
513 QFileInfo fInfo(directory + QDir::separator() + name);
514 return fInfo.lastModified().toTime_t();
515}
516
517//static
518int64_t FSDirectory::fileModified(const QString& dir, const QString& name)
519{
520 QFileInfo fInfo(dir + QDir::separator() + name);
521 return fInfo.lastModified().toTime_t();
522}
523
524void FSDirectory::touchFile(const QString& name)
525{
526 CND_PRECONDITION(!directory.isEmpty(), "directory is not open");
527
528 QFile file(directory + QDir::separator() + name);
529 if (!file.open(QIODevice::ReadWrite))
530 _CLTHROWA(CL_ERR_IO, "IO Error while touching file");
531}
532
533int64_t FSDirectory::fileLength(const QString& name) const
534{
535 CND_PRECONDITION(!directory.isEmpty(), "directory is not open");
536
537 QFileInfo fInfo(directory + QDir::separator() + name);
538 return fInfo.size();
539}
540
541IndexInput* FSDirectory::openInput(const QString& name)
542{
543 return openInput(name, CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE);
544}
545
546IndexInput* FSDirectory::openInput(const QString& name, int32_t bufferSize )
547{
548 CND_PRECONDITION(directory[0]!=0,"directory is not open")
549
550 return _CLNEW FSIndexInput(directory + QDir::separator() + name, bufferSize);
551}
552
553void FSDirectory::close()
554{
555 SCOPED_LOCK_MUTEX(DIRECTORIES.THIS_LOCK)
556 {
557 SCOPED_LOCK_MUTEX(THIS_LOCK)
558
559 CND_PRECONDITION(!directory.isEmpty(), "directory is not open");
560
561 //refcount starts at 1
562 if (--refCount <= 0) {
563 Directory* dir = DIRECTORIES.get(getDirName());
564 if (dir) {
565 //this will be removed in ~FSDirectory
566 DIRECTORIES.remove(getDirName());
567 _CLDECDELETE(dir);
568 }
569 }
570 }
571}
572
573QString FSDirectory::getLockPrefix() const
574{
575 CND_PRECONDITION(!directory.isEmpty(), "directory is not open");
576
577 QString dirName(QFileInfo(directory).absoluteFilePath());
578 if (dirName.isEmpty())
579 _CLTHROWA(CL_ERR_Runtime, "Invalid directory path");
580
581 // to be compatible with jlucene,
582 // we need to make some changes ...
583 if (dirName.at(1) == QLatin1Char(':'))
584 dirName[0] = dirName.at(0).toUpper();
585
586 TCHAR tBuffer[2048] = { 0 };
587 dirName.toWCharArray(tBuffer);
588
589 char aBuffer[4096] = { 0 };
590 STRCPY_TtoA(aBuffer, tBuffer, 4096);
591
592 QString string(QLatin1String("lucene-"));
593 QByteArray hash(QCryptographicHash::hash(aBuffer, QCryptographicHash::Md5));
594
595 // TODO: verify this !!!
596 return string.append(QLatin1String(hash.toHex().constData()));
597}
598
599bool FSDirectory::doDeleteFile(const QString& name)
600{
601 CND_PRECONDITION(!directory.isEmpty(), "directory is not open");
602
603 QDir dir(directory);
604 return dir.remove(name);
605}
606
607void FSDirectory::renameFile(const QString& from, const QString& to)
608{
609 CND_PRECONDITION(!directory.isEmpty(), "directory is not open");
610 SCOPED_LOCK_MUTEX(THIS_LOCK)
611
612 if (fileExists(to))
613 deleteFile(to, false);
614
615 QFile file(directory + QDir::separator() + from);
616 QString newFile(directory + QDir::separator() + to);
617 if (!file.rename(newFile)) {
618 // try a second time if we fail
619 if (fileExists(to))
620 deleteFile(to, false);
621
622 if (!file.rename(newFile)) {
623 QString error(QLatin1String("Could not rename: %1 to %2!!!!"));
624 error.arg(from).arg(newFile);
625 QByteArray bArray(error.toLocal8Bit());
626 _CLTHROWA(CL_ERR_IO, bArray.constData());
627 }
628 }
629}
630
631IndexOutput* FSDirectory::createOutput(const QString& name)
632{
633 CND_PRECONDITION(!directory.isEmpty(), "directory is not open");
634
635 QString file = directory + QDir::separator() + name;
636 if (QFileInfo(file).exists()) {
637 if (!QFile::remove(file)) {
638 QByteArray bArray("Cannot overwrite: ");
639 bArray.append(name.toLocal8Bit());
640 _CLTHROWA(CL_ERR_IO, bArray.constData());
641 }
642 }
643 return _CLNEW FSIndexOutput(file);
644}
645
646LuceneLock* FSDirectory::makeLock(const QString& name)
647{
648 CND_PRECONDITION(!directory.isEmpty(), "directory is not open");
649
650
651 QString lockFile(getLockPrefix());
652 lockFile.append(QLatin1Char('-')).append(name);
653
654 return _CLNEW FSLock(lockDir, lockFile);
655}
656
657QString FSDirectory::toString() const
658{
659 return QString::fromLatin1("FSDirectory@").append(directory);
660}
661
662CL_NS_END
663