Fix compilation
[kdevelop:devel-kdevplatform.git] / language / backgroundparser / backgroundparser.cpp
1 /*
2  * This file is part of KDevelop
3  *
4  * Copyright 2006 Adam Treat <treat@kde.org>
5  * Copyright 2007 Kris Wong <kris.p.wong@gmail.com>
6  * Copyright 2007-2008 David Nolden <david.nolden.kdevelop@art-master.de>
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU Library General Public License as
10  * published by the Free Software Foundation; either version 2 of the
11  * License, or (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful,
14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
16  * GNU General Public License for more details.
17  *
18  * You should have received a copy of the GNU General Public
19  * License along with this program; if not, write to the
20  * Free Software Foundation, Inc.,
21  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
22  */
23
24 #include "backgroundparser.h"
25
26 #include <QList>
27 #include <QFile>
28 #include <QTimer>
29 #include <QMutex>
30 #include <QWaitCondition>
31 #include <QMutexLocker>
32 #include <QThread>
33
34 #include <kdebug.h>
35 #include <kglobal.h>
36 #include <kconfiggroup.h>
37 #include <ksharedconfig.h>
38 #include <klocale.h>
39
40 #include <ktexteditor/smartrange.h>
41 #include <ktexteditor/smartinterface.h>
42 #include <ktexteditor/document.h>
43
44 #include <threadweaver/State.h>
45 #include <threadweaver/ThreadWeaver.h>
46 #include <threadweaver/JobCollection.h>
47 #include <threadweaver/DebuggingAids.h>
48
49 #include <interfaces/ilanguagecontroller.h>
50 #include <interfaces/ilanguage.h>
51
52 #include "../interfaces/ilanguagesupport.h"
53
54 #include "parsejob.h"
55 #include "parserdependencypolicy.h"
56 #include <editor/modificationrevisionset.h>
57 #include <interfaces/icore.h>
58 #include <qcoreapplication.h>
59 #include <interfaces/idocumentcontroller.h>
60
61 const bool separateThreadForHighPriority = true;
62
63 namespace KDevelop
64 {
65
66 class BackgroundParserPrivate
67 {
68 public:
69     BackgroundParserPrivate(BackgroundParser *parser, ILanguageController *languageController)
70         :m_parser(parser), m_languageController(languageController), m_shuttingDown(false), m_mutex(QMutex::Recursive)
71     {
72         parser->d = this; //Set this so we can safely call back BackgroundParser from within loadSettings()
73
74         m_timer.setSingleShot(true);
75         m_delay = 500;
76         m_threads = 1;
77         m_doneParseJobs = 0;
78         m_maxParseJobs = 0;
79         m_neededPriority = BackgroundParser::WorstPriority;
80
81         ThreadWeaver::setDebugLevel(true, 1);
82
83         QObject::connect(&m_timer, SIGNAL(timeout()), m_parser, SLOT(parseDocuments()));
84
85         loadSettings(); // Start the weaver
86     }
87
88     void startTimerThreadSafe() {
89         QMetaObject::invokeMethod(m_parser, "startTimer", Qt::QueuedConnection);
90     }
91
92     ~BackgroundParserPrivate()
93     {
94         suspend();
95
96         m_weaver.dequeue();
97         m_weaver.requestAbort();
98         m_weaver.finish();
99
100         // Release dequeued jobs
101         QHashIterator<KUrl, ParseJob*> it = m_parseJobs;
102         while (it.hasNext()) {
103             it.next();
104             it.value()->setBackgroundParser(0);
105             delete it.value();
106         }
107     }
108
109     // Non-mutex guarded functions, only call with m_mutex acquired.
110     void parseDocumentsInternal()
111     {
112         if(m_shuttingDown)
113             return;
114         // Create delayed jobs, that is, jobs for documents which have been changed
115         // by the user.
116         QList<ParseJob*> jobs;
117
118         for (QMap<int, QSet<KUrl> >::Iterator it1 = m_documentsForPriority.begin();
119              it1 != m_documentsForPriority.end(); ++it1 )
120         {
121             if(it1.key() > m_neededPriority)
122                 break; //The priority is not good enough to be processed right now
123
124             for(QSet<KUrl>::Iterator it = it1.value().begin(); it != it1.value().end();) {
125                 //Only create parse-jobs for up to thread-count * 2 documents, so we don't fill the memory unnecessarily
126                 if(m_parseJobs.count() >= m_threads+1 || (m_parseJobs.count() >= m_threads && !separateThreadForHighPriority) )
127                     break;
128
129                 if(m_parseJobs.count() >= m_threads && it1.key() > BackgroundParser::NormalPriority && !specialParseJob)
130                     break; //The additional parsing thread is reserved for higher priority parsing
131
132                 // When a document is scheduled for parsing while it is being parsed, it will be parsed
133                 // again once the job finished, but not now.
134                 if (m_parseJobs.contains(*it) ) {
135                     ++it;
136                     continue;
137                 }
138
139                 kDebug(9505) << "creating parse-job" << *it << "new count of active parse-jobs:" << m_parseJobs.count() + 1;
140                 ParseJob* job = createParseJob(*it, m_documents[*it].features(), m_documents[*it].notifyWhenReady());
141
142                 if(m_parseJobs.count() == m_threads+1 && !specialParseJob)
143                     specialParseJob = job; //This parse-job is allocated into the reserved thread
144
145                 if(job)
146                     jobs.append(job);
147
148                 m_documents.remove(*it);
149                 it = it1.value().erase(it);
150                 --m_maxParseJobs; //We have added one when putting the document into m_documents
151             }
152         }
153
154         // Ok, enqueueing is fine because m_parseJobs contains all of the jobs now
155
156         foreach (ParseJob* job, jobs)
157             m_weaver.enqueue(job);
158
159         m_parser->updateProgressBar();
160
161         //We don't hide the progress-bar in updateProgressBar, so it doesn't permanently flash when a document is reparsed again and again.
162         if(m_doneParseJobs == m_maxParseJobs)
163             emit m_parser->hideProgress(m_parser);
164     }
165
166     ParseJob* createParseJob(const KUrl& url, TopDUContext::Features features, QList<QPointer<QObject> > notifyWhenReady)
167     {
168         QList<ILanguage*> languages = m_languageController->languagesForUrl(url);
169         foreach (ILanguage* language, languages) {
170             if(!language) {
171                 kWarning() << "got zero language for" << url;
172                 continue;
173             }
174             if(!language->languageSupport()) {
175                 kWarning() << "language has no language support assigned:" << language->name();
176                 continue;
177             }
178             ParseJob* job = language->languageSupport()->createParseJob(url);
179             if (!job) {
180                 continue; // Language part did not produce a valid ParseJob.
181             }
182
183             job->setMinimumFeatures(features);
184             job->setBackgroundParser(m_parser);
185             job->setNotifyWhenReady(notifyWhenReady);
186             job->setTracker(m_parser->trackerForUrl(IndexedString(url)));
187
188             QObject::connect(job, SIGNAL(done(ThreadWeaver::Job*)),
189                                 m_parser, SLOT(parseComplete(ThreadWeaver::Job*)));
190             QObject::connect(job, SIGNAL(failed(ThreadWeaver::Job*)),
191                                 m_parser, SLOT(parseComplete(ThreadWeaver::Job*)));
192             QObject::connect(job, SIGNAL(progress(KDevelop::ParseJob*, float, QString)),
193                                 m_parser, SLOT(parseProgress(KDevelop::ParseJob*, float, QString)), Qt::QueuedConnection);
194
195             m_parseJobs.insert(url, job);
196
197             ++m_maxParseJobs;
198
199             // TODO more thinking required here to support multiple parse jobs per url (where multiple language plugins want to parse)
200             return job;
201         }
202
203         if(languages.isEmpty())
204             kDebug() << "found no languages for url" << url;
205         else
206             kDebug() << "could not create parse-job for url" << url;
207
208         //Notify that we failed
209         typedef QPointer<QObject> Notify;
210         foreach(const Notify& n, notifyWhenReady)
211             if(n)
212                 QMetaObject::invokeMethod(n, "updateReady", Qt::QueuedConnection, Q_ARG(KDevelop::IndexedString, IndexedString(url)), Q_ARG(KDevelop::ReferencedTopDUContext, ReferencedTopDUContext()));
213
214         return 0;
215     }
216
217
218     void loadSettings()
219     {
220         ///@todo re-load settings when they have been changed!
221         KConfigGroup config(KGlobal::config(), "Background Parser");
222
223         m_delay = config.readEntry("Delay", 500);
224         m_timer.setInterval(m_delay);
225         m_threads = 0;
226         m_parser->setThreadCount(config.readEntry("Number of Threads", 1));
227
228         if (config.readEntry("Enabled", true)) {
229             resume();
230         } else {
231             suspend();
232         }
233     }
234
235     void suspend()
236     {
237         bool s = m_weaver.state().stateId() == ThreadWeaver::Suspended ||
238                  m_weaver.state().stateId() == ThreadWeaver::Suspending;
239
240         if (s) { // Already suspending
241             return;
242         }
243
244         m_timer.stop();
245         m_weaver.suspend();
246     }
247
248     void resume()
249     {
250         bool s = m_weaver.state().stateId() == ThreadWeaver::Suspended ||
251                  m_weaver.state().stateId() == ThreadWeaver::Suspending;
252
253         if (m_timer.isActive() && !s) { // Not suspending
254             return;
255         }
256
257         m_timer.start(m_delay);
258         m_weaver.resume();
259     }
260
261     BackgroundParser *m_parser;
262     ILanguageController* m_languageController;
263
264     //Current parse-job that is executed in the additional thread
265     QPointer<ParseJob> specialParseJob;
266
267     QTimer m_timer;
268     int m_delay;
269     int m_threads;
270     
271     bool m_shuttingDown;
272     
273     struct DocumentParseTarget {
274         QPointer<QObject> notifyWhenReady;
275         int priority;
276         TopDUContext::Features features;
277         bool operator==(const DocumentParseTarget& rhs) const {
278             return notifyWhenReady == rhs.notifyWhenReady && priority == rhs.priority && features == rhs.features;
279         }
280     };
281
282     struct DocumentParsePlan {
283         QList<DocumentParseTarget> targets;
284
285         int priority() const {
286             //Pick the best priority
287             int ret = BackgroundParser::WorstPriority;
288             foreach(const DocumentParseTarget &target, targets)
289                 if(target.priority < ret)
290                     ret = target.priority;
291             return ret;
292         }
293
294         TopDUContext::Features features() const {
295             //Pick the best features
296             TopDUContext::Features ret = (TopDUContext::Features)0;
297             foreach(const DocumentParseTarget &target, targets)
298                 ret = (TopDUContext::Features) (ret | target.features);
299             return ret;
300         }
301
302         QList<QPointer<QObject> > notifyWhenReady() {
303             QList<QPointer<QObject> > ret;
304
305             foreach(const DocumentParseTarget &target, targets)
306                 if(target.notifyWhenReady)
307                     ret << target.notifyWhenReady;
308
309             return ret;
310         }
311     };
312     // A list of documents that are planned to be parsed, and their priority
313     QMap<KUrl, DocumentParsePlan > m_documents;
314     // The documents ordered by priority
315     QMap<int, QSet<KUrl> > m_documentsForPriority;
316     // Currently running parse jobs
317     QHash<KUrl, ParseJob*> m_parseJobs;
318     // A change tracker for each managed document
319     QHash<IndexedString, DocumentChangeTracker*> m_managed;
320     // The url for each managed document. Those may temporarily differ from the real url.
321     QHash<KTextEditor::Document*, IndexedString> m_managedTextDocumentUrls;
322
323     ThreadWeaver::Weaver m_weaver;
324     ParserDependencyPolicy m_dependencyPolicy;
325
326     QMutex m_mutex;
327
328     int m_maxParseJobs;
329     int m_doneParseJobs;
330     QMap<KDevelop::ParseJob*, float> m_jobProgress;
331     int m_neededPriority; //The minimum priority needed for processed jobs
332 };
333
334 BackgroundParser::BackgroundParser(ILanguageController *languageController)
335     : QObject(languageController), d(new BackgroundParserPrivate(this, languageController))
336 {
337     Q_ASSERT(ICore::self()->documentController());
338     connect(ICore::self()->documentController(), SIGNAL(documentLoaded(KDevelop::IDocument*)), this, SLOT(documentLoaded(KDevelop::IDocument*)));
339     connect(ICore::self()->documentController(), SIGNAL(documentUrlChanged(KDevelop::IDocument*)), this, SLOT(documentUrlChanged(KDevelop::IDocument*)));
340     connect(ICore::self()->documentController(), SIGNAL(documentClosed(KDevelop::IDocument*)), this, SLOT(documentClosed(KDevelop::IDocument*)));
341     connect(QCoreApplication::instance(), SIGNAL(aboutToQuit()), this, SLOT(aboutToQuit()));
342 }
343
344 void BackgroundParser::aboutToQuit()
345 {
346     d->m_shuttingDown = true;
347 }
348
349 BackgroundParser::~BackgroundParser()
350 {
351     delete d;
352 }
353
354 QString BackgroundParser::statusName() const
355 {
356     return i18n("Background Parser");
357 }
358
359 void BackgroundParser::clear(QObject* parent)
360 {
361     QMutexLocker lock(&d->m_mutex);
362
363     QHashIterator<KUrl, ParseJob*> it = d->m_parseJobs;
364     while (it.hasNext()) {
365         it.next();
366         if (it.value()->parent() == parent) {
367             it.value()->requestAbort();
368         }
369     }
370 }
371
372 void BackgroundParser::loadSettings(bool projectIsLoaded)
373 {
374     Q_UNUSED(projectIsLoaded)
375
376
377     d->loadSettings();
378 }
379
380 void BackgroundParser::saveSettings(bool projectIsLoaded)
381 {
382     Q_UNUSED(projectIsLoaded)
383 }
384
385 void BackgroundParser::parseProgress(KDevelop::ParseJob* job, float value, QString text)
386 {
387     Q_UNUSED(text)
388     d->m_jobProgress[job] = value;
389     updateProgressBar();
390 }
391
392 void BackgroundParser::revertAllRequests(QObject* notifyWhenReady)
393 {
394     QPointer<QObject> p(notifyWhenReady);
395
396     QMutexLocker lock(&d->m_mutex);
397     for(QMap<KUrl, BackgroundParserPrivate::DocumentParsePlan >::iterator it = d->m_documents.begin(); it != d->m_documents.end(); ) {
398
399         d->m_documentsForPriority[it.value().priority()].remove(it.key());
400
401         int index = -1;
402         for(int a = 0; a < (*it).targets.size(); ++a)
403             if((*it).targets[a].notifyWhenReady == notifyWhenReady)
404                 index = a;
405
406         if(index != -1) {
407             (*it).targets.removeAt(index);
408             if((*it).targets.isEmpty()) {
409                 it = d->m_documents.erase(it);
410                 --d->m_maxParseJobs;
411
412                 continue;
413             }
414         }
415
416         d->m_documentsForPriority[it.value().priority()].insert(it.key());
417         ++it;
418     }
419 }
420
421 void BackgroundParser::addDocument(const KUrl& url, TopDUContext::Features features, int priority, QObject* notifyWhenReady)
422 {
423 //     kDebug(9505) << "BackgroundParser::addDocument" << url.prettyUrl();
424     QMutexLocker lock(&d->m_mutex);
425     {
426         Q_ASSERT(url.isValid());
427
428         BackgroundParserPrivate::DocumentParseTarget target;
429         target.priority = priority;
430         target.features = features;
431         target.notifyWhenReady = QPointer<QObject>(notifyWhenReady);
432
433         QMap<KUrl, BackgroundParserPrivate::DocumentParsePlan>::iterator it = d->m_documents.find(url);
434
435         if (it != d->m_documents.end()) {
436             //Update the stored plan
437
438             d->m_documentsForPriority[it.value().priority()].remove(url);
439             it.value().targets << target;
440             d->m_documentsForPriority[it.value().priority()].insert(url);
441         }else{
442 //             kDebug(9505) << "BackgroundParser::addDocument: queuing" << url;
443             d->m_documents[url].targets << target;
444             d->m_documentsForPriority[d->m_documents[url].priority()].insert(url);
445             ++d->m_maxParseJobs; //So the progress-bar waits for this document
446         }
447
448         d->startTimerThreadSafe();
449     }
450 }
451
452 void BackgroundParser::addDocumentList(const KUrl::List &urls, TopDUContext::Features features, int priority)
453 {
454     foreach (const KUrl &url, urls)
455         addDocument(url, features, priority);
456 }
457
458 void BackgroundParser::removeDocument(const KUrl &url, QObject* notifyWhenReady)
459 {
460     QMutexLocker lock(&d->m_mutex);
461
462     Q_ASSERT(url.isValid());
463
464     if(d->m_documents.contains(url)) {
465         
466         d->m_documentsForPriority[d->m_documents[url].priority()].remove(url);
467         
468         foreach(BackgroundParserPrivate::DocumentParseTarget target, d->m_documents[url].targets)
469             if(target.notifyWhenReady == notifyWhenReady)
470                 d->m_documents[url].targets.removeAll(target);
471
472         if(d->m_documents[url].targets.isEmpty()) {
473             d->m_documents.remove(url);
474             --d->m_maxParseJobs;
475         }else{
476             //Insert with an eventually different priority
477             d->m_documentsForPriority[d->m_documents[url].priority()].insert(url);
478         }
479     }
480 }
481
482 void BackgroundParser::parseDocuments()
483 {
484     QMutexLocker lock(&d->m_mutex);
485
486     d->parseDocumentsInternal();
487 }
488
489 void BackgroundParser::parseComplete(ThreadWeaver::Job* job)
490 {
491     if (ParseJob* parseJob = qobject_cast<ParseJob*>(job)) {
492
493         emit parseJobFinished(parseJob);
494
495         {
496             {
497                 QMutexLocker lock(&d->m_mutex);
498     
499                 d->m_parseJobs.remove(parseJob->document().str());
500     
501                 d->m_jobProgress.remove(parseJob);
502     
503                 parseJob->setBackgroundParser(0);
504     
505                 ++d->m_doneParseJobs;
506                 updateProgressBar();
507             }
508             //Unlock the mutex before deleting the parse-job, because the parse-job
509             //has a virtual destructor that may lock the duchain, leading to deadlocks
510             delete parseJob;
511         }
512         //Continue creating more parse-jobs
513         QMetaObject::invokeMethod(this, "parseDocuments", Qt::QueuedConnection);
514     }
515 }
516
517 void BackgroundParser::disableProcessing()
518 {
519     setNeededPriority(BestPriority);
520 }
521
522 void BackgroundParser::enableProcessing()
523 {
524     setNeededPriority(WorstPriority);
525 }
526
527 bool BackgroundParser::isQueued(KUrl url) const {
528     QMutexLocker lock(&d->m_mutex);
529     return d->m_documents.contains(url);
530 }
531
532 int BackgroundParser::queuedCount() const
533 {
534     QMutexLocker lock(&d->m_mutex);
535     return d->m_documents.count();
536 }
537
538 void BackgroundParser::setNeededPriority(int priority)
539 {
540     QMutexLocker lock(&d->m_mutex);
541     d->m_neededPriority = priority;
542     d->startTimerThreadSafe();
543 }
544
545 void BackgroundParser::suspend()
546 {
547     d->suspend();
548
549     emit hideProgress(this);
550 }
551
552 void BackgroundParser::resume()
553 {
554     d->resume();
555
556     updateProgressBar();
557 }
558
559 void BackgroundParser::updateProgressBar()
560 {
561     if (d->m_doneParseJobs >= d->m_maxParseJobs) {
562         if(d->m_doneParseJobs > d->m_maxParseJobs) {
563             kDebug() << "m_doneParseJobs larger than m_maxParseJobs:" << d->m_doneParseJobs << d->m_maxParseJobs;
564         }
565         d->m_doneParseJobs = 0;
566         d->m_maxParseJobs = 0;
567     } else {
568         float additionalProgress = 0;
569         for(QMap<KDevelop::ParseJob*, float>::const_iterator it = d->m_jobProgress.constBegin(); it != d->m_jobProgress.constEnd(); ++it)
570             additionalProgress += *it;
571
572         emit showProgress(this, 0, d->m_maxParseJobs*1000, (additionalProgress + d->m_doneParseJobs)*1000);
573     }
574 }
575
576 ParserDependencyPolicy* BackgroundParser::dependencyPolicy() const
577 {
578     return &d->m_dependencyPolicy;
579 }
580
581 ParseJob* BackgroundParser::parseJobForDocument(const KUrl& document) const
582 {
583     QMutexLocker lock(&d->m_mutex);
584
585     if (d->m_parseJobs.contains(document)) {
586         return d->m_parseJobs[document];
587     }
588
589     return 0;
590 }
591
592 void BackgroundParser::setThreadCount(int threadCount)
593 {
594     if (d->m_threads != threadCount) {
595         d->m_threads = threadCount;
596         d->m_weaver.setMaximumNumberOfThreads(d->m_threads+1); //1 Additional thread for high-priority parsing
597     }
598 }
599
600 void BackgroundParser::setDelay(int miliseconds)
601 {
602     if (d->m_delay != miliseconds) {
603         d->m_delay = miliseconds;
604         d->m_timer.setInterval(d->m_delay);
605     }
606 }
607
608 QList< IndexedString > BackgroundParser::managedDocuments()
609 {
610     QMutexLocker l(&d->m_mutex);
611     
612     return d->m_managed.keys();
613 }
614
615 DocumentChangeTracker* BackgroundParser::trackerForUrl(const KDevelop::IndexedString& url) const
616 {
617     QMutexLocker l(&d->m_mutex);
618     
619     QHash< IndexedString, DocumentChangeTracker* >::iterator it = d->m_managed.find(url);
620     if(it != d->m_managed.end())
621         return *it;
622     else
623         return 0;
624 }
625
626 void BackgroundParser::documentClosed ( IDocument* document )
627 {
628     QMutexLocker l(&d->m_mutex);
629     
630     if(document->textDocument())
631     {
632         KTextEditor::Document* textDocument = document->textDocument();
633         
634         if(!d->m_managedTextDocumentUrls.contains(textDocument))
635             return; // Probably the document had an invalid url, and thus it wasn't added to the background parser
636         
637         Q_ASSERT(d->m_managedTextDocumentUrls.contains(textDocument));
638         
639         IndexedString url(d->m_managedTextDocumentUrls[textDocument]);
640         Q_ASSERT(d->m_managed.contains(url));
641         
642         kDebug() << "removing" << url.str() << "from background parser";
643         delete d->m_managed[url];
644         d->m_managedTextDocumentUrls.remove(textDocument);
645         d->m_managed.remove(url);
646     }
647 }
648
649 void BackgroundParser::documentLoaded( IDocument* document )
650 {
651     QMutexLocker l(&d->m_mutex);
652     if(document->textDocument() && document->textDocument()->url().isValid())
653     {
654         KTextEditor::Document* textDocument = document->textDocument();
655         
656         kDebug() << "Creating change tracker for " << document->url();
657         IndexedString url(document->url());
658         // Some debugging because we had issues with this
659         if(d->m_managed.contains(url))
660             Q_ASSERT(d->m_managed[url]->document() == textDocument);
661         
662         Q_ASSERT(!d->m_managed.contains(url));
663         Q_ASSERT(!d->m_managedTextDocumentUrls.contains(textDocument));
664         
665         d->m_managedTextDocumentUrls[textDocument] = url;
666         d->m_managed.insert(url, new DocumentChangeTracker(textDocument));
667     }else{
668         kDebug() << "NOT creating change tracker for" << document->url();
669     }
670 }
671
672 void BackgroundParser::documentUrlChanged(IDocument* document)
673 {
674     documentClosed(document);
675     
676     // Only call documentLoaded if the file wasn't renamed to a filename that is already tracked.
677     if(document->textDocument() && !d->m_managed.contains(IndexedString(document->textDocument()->url())))
678         documentLoaded(document);
679 }
680
681 void BackgroundParser::startTimer() {
682     d->m_timer.start(d->m_delay);
683 }
684
685 }
686
687 #include "backgroundparser.moc"
688