Don't ignore source-text when generating qsTrId translations for QML
[qt:qt.git] / tools / linguist / lupdate / qdeclarative.cpp
1 /****************************************************************************
2 **
3 ** Copyright (C) 2011 Nokia Corporation and/or its subsidiary(-ies).
4 ** All rights reserved.
5 ** Contact: Nokia Corporation (qt-info@nokia.com)
6 **
7 ** This file is part of the Qt Linguist of the Qt Toolkit.
8 **
9 ** $QT_BEGIN_LICENSE:LGPL$
10 ** No Commercial Usage
11 ** This file contains pre-release code and may not be distributed.
12 ** You may use this file in accordance with the terms and conditions
13 ** contained in the Technology Preview License Agreement accompanying
14 ** this package.
15 **
16 ** GNU Lesser General Public License Usage
17 ** Alternatively, this file may be used under the terms of the GNU Lesser
18 ** General Public License version 2.1 as published by the Free Software
19 ** Foundation and appearing in the file LICENSE.LGPL included in the
20 ** packaging of this file.  Please review the following information to
21 ** ensure the GNU Lesser General Public License version 2.1 requirements
22 ** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
23 **
24 ** In addition, as a special exception, Nokia gives you certain additional
25 ** rights.  These rights are described in the Nokia Qt LGPL Exception
26 ** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
27 **
28 ** If you have questions regarding the use of this file, please contact
29 ** Nokia at qt-info@nokia.com.
30 **
31 **
32 **
33 **
34 **
35 **
36 **
37 **
38 ** $QT_END_LICENSE$
39 **
40 ****************************************************************************/
41
42 #include "lupdate.h"
43
44 #include <translator.h>
45
46 #include <QtCore/QDebug>
47 #include <QtCore/QFile>
48 #include <QtCore/QString>
49
50 #include "parser/qdeclarativejsengine_p.h"
51 #include "parser/qdeclarativejsparser_p.h"
52 #include "parser/qdeclarativejslexer_p.h"
53 #include "parser/qdeclarativejsnodepool_p.h"
54 #include "parser/qdeclarativejsastvisitor_p.h"
55 #include "parser/qdeclarativejsast_p.h"
56
57 #include <QCoreApplication>
58 #include <QFile>
59 #include <QFileInfo>
60 #include <QtDebug>
61 #include <QStringList>
62
63 #include <iostream>
64 #include <cstdlib>
65
66 QT_BEGIN_NAMESPACE
67
68 class LU {
69     Q_DECLARE_TR_FUNCTIONS(LUpdate)
70 };
71
72 using namespace QDeclarativeJS;
73
74 class Comment
75 {
76 public:
77     Comment() : lastLine(-1) {}
78     QString extracomment;
79     QString msgid;
80     TranslatorMessage::ExtraData extra;
81     QString sourcetext;
82     int lastLine;
83
84     bool isValid() const
85     { return !extracomment.isEmpty() || !msgid.isEmpty() || !sourcetext.isEmpty() || !extra.isEmpty(); }
86 };
87
88 class FindTrCalls: protected AST::Visitor
89 {
90 public:
91     void operator()(Translator *translator, const QString &fileName, AST::Node *node)
92     {
93         m_translator = translator;
94         m_fileName = fileName;
95         m_component = QFileInfo(fileName).baseName();   //matches qsTr usage in QScriptEngine
96         accept(node);
97     }
98
99     QList<Comment> comments;
100
101 protected:
102     using AST::Visitor::visit;
103     using AST::Visitor::endVisit;
104
105     void accept(AST::Node *node)
106     { AST::Node::acceptChild(node, this); }
107
108     virtual void endVisit(AST::CallExpression *node)
109     {
110         m_bSource.clear();
111         if (AST::IdentifierExpression *idExpr = AST::cast<AST::IdentifierExpression *>(node->base)) {
112             if (idExpr->name->asString() == QLatin1String("qsTr") ||
113                 idExpr->name->asString() == QLatin1String("QT_TR_NOOP")) {
114                 if (!node->arguments)
115                     return;
116                 AST::BinaryExpression *binary = AST::cast<AST::BinaryExpression *>(node->arguments->expression);
117                 if (binary) {
118                     if (!createString(binary))
119                         m_bSource.clear();
120                 }
121                 AST::StringLiteral *literal = AST::cast<AST::StringLiteral *>(node->arguments->expression);
122                 if (literal || !m_bSource.isEmpty()) {
123                     const QString source = literal ? literal->value->asString() : m_bSource;
124
125                     QString comment;
126                     bool plural = false;
127                     AST::ArgumentList *commentNode = node->arguments->next;
128                     if (commentNode && AST::cast<AST::StringLiteral *>(commentNode->expression)) {
129                         literal = AST::cast<AST::StringLiteral *>(commentNode->expression);
130                         comment = literal->value->asString();
131
132                         AST::ArgumentList *nNode = commentNode->next;
133                         if (nNode)
134                             plural = true;
135                     }
136
137                     QString id;
138                     QString extracomment;
139                     TranslatorMessage::ExtraData extra;
140                     Comment scomment = findComment(node->firstSourceLocation().startLine);
141                     if (scomment.isValid()) {
142                         extracomment = scomment.extracomment;
143                         extra = scomment.extra;
144                         id = scomment.msgid;
145                     }
146
147                     TranslatorMessage msg(m_component, source,
148                         comment, QString(), m_fileName,
149                         node->firstSourceLocation().startLine, QStringList(),
150                         TranslatorMessage::Unfinished, plural);
151                     msg.setExtraComment(extracomment.simplified());
152                     msg.setId(id);
153                     msg.setExtras(extra);
154                     m_translator->extend(msg);
155                 }
156             } else if (idExpr->name->asString() == QLatin1String("qsTranslate") ||
157                        idExpr->name->asString() == QLatin1String("QT_TRANSLATE_NOOP")) {
158                 if (node->arguments && AST::cast<AST::StringLiteral *>(node->arguments->expression)) {
159                     AST::StringLiteral *literal = AST::cast<AST::StringLiteral *>(node->arguments->expression);
160                     const QString context = literal->value->asString();
161
162                     QString source;
163                     QString comment;
164                     bool plural = false;
165                     AST::ArgumentList *sourceNode = node->arguments->next;
166                     if (!sourceNode)
167                         return;
168                     literal = AST::cast<AST::StringLiteral *>(sourceNode->expression);
169                     AST::BinaryExpression *binary = AST::cast<AST::BinaryExpression *>(sourceNode->expression);
170                     if (binary) {
171                         if (!createString(binary))
172                             m_bSource.clear();
173                     }
174                     if (!literal && m_bSource.isEmpty())
175                         return;
176
177                     QString id;
178                     QString extracomment;
179                     TranslatorMessage::ExtraData extra;
180                     Comment scomment = findComment(node->firstSourceLocation().startLine);
181                     if (scomment.isValid()) {
182                         extracomment = scomment.extracomment;
183                         extra = scomment.extra;
184                         id = scomment.msgid;
185                     }
186
187                     source = literal ? literal->value->asString() : m_bSource;
188                     AST::ArgumentList *commentNode = sourceNode->next;
189                     if (commentNode && AST::cast<AST::StringLiteral *>(commentNode->expression)) {
190                         literal = AST::cast<AST::StringLiteral *>(commentNode->expression);
191                         comment = literal->value->asString();
192
193                         AST::ArgumentList *nNode = commentNode->next;
194                         if (nNode)
195                             plural = true;
196                     }
197
198                     TranslatorMessage msg(context, source,
199                         comment, QString(), m_fileName,
200                         node->firstSourceLocation().startLine, QStringList(),
201                         TranslatorMessage::Unfinished, plural);
202                     msg.setExtraComment(extracomment.simplified());
203                     msg.setId(id);
204                     msg.setExtras(extra);
205                     m_translator->extend(msg);
206                 }
207             } else if (idExpr->name->asString() == QLatin1String("qsTrId") ||
208                        idExpr->name->asString() == QLatin1String("QT_TRID_NOOP")) {
209                 if (!node->arguments)
210                     return;
211
212                 AST::StringLiteral *literal = AST::cast<AST::StringLiteral *>(node->arguments->expression);
213                 if (literal) {
214
215                     QString extracomment;
216                     QString sourcetext;
217                     TranslatorMessage::ExtraData extra;
218                     Comment comment = findComment(node->firstSourceLocation().startLine);
219                     if (comment.isValid()) {
220                         extracomment = comment.extracomment;
221                         sourcetext = comment.sourcetext;
222                         extra = comment.extra;
223                     }
224
225                     const QString id = literal->value->asString();
226                     bool plural = node->arguments->next;
227
228                     TranslatorMessage msg(QString(), sourcetext,
229                         QString(), QString(), m_fileName,
230                         node->firstSourceLocation().startLine, QStringList(),
231                         TranslatorMessage::Unfinished, plural);
232                     msg.setExtraComment(extracomment.simplified());
233                     msg.setId(id);
234                     msg.setExtras(extra);
235                     m_translator->extend(msg);
236                 }
237             }
238         }
239     }
240
241 private:
242     bool createString(AST::BinaryExpression *b)
243     {
244         if (!b || b->op != 0)
245             return false;
246         AST::BinaryExpression *l = AST::cast<AST::BinaryExpression *>(b->left);
247         AST::BinaryExpression *r = AST::cast<AST::BinaryExpression *>(b->right);
248         AST::StringLiteral *ls = AST::cast<AST::StringLiteral *>(b->left);
249         AST::StringLiteral *rs = AST::cast<AST::StringLiteral *>(b->right);
250         if ((!l && !ls) || (!r && !rs))
251             return false;
252         if (l) {
253             if (!createString(l))
254                 return false;
255         } else
256             m_bSource.prepend(ls->value->asString());
257
258         if (r) {
259             if (!createString(r))
260                 return false;
261         } else
262             m_bSource.append(rs->value->asString());
263
264         return true;
265     }
266
267     Comment findComment(int loc)
268     {
269         if (comments.isEmpty())
270             return Comment();
271
272         int i = 0;
273         int commentLoc = comments.at(i).lastLine;
274         while (commentLoc <= loc) {
275             if (commentLoc == loc)
276                 return comments.at(i);
277             if (i == comments.count()-1)
278                 break;
279             commentLoc = comments.at(++i).lastLine;
280         }
281         return Comment();
282     }
283
284     Translator *m_translator;
285     QString m_fileName;
286     QString m_component;
287     QString m_bSource;
288 };
289
290 QString createErrorString(const QString &filename, const QString &code, Parser &parser)
291 {
292     // print out error
293     QStringList lines = code.split(QLatin1Char('\n'));
294     lines.append(QLatin1String("\n")); // sentinel.
295     QString errorString;
296
297     foreach (const DiagnosticMessage &m, parser.diagnosticMessages()) {
298
299         if (m.isWarning())
300             continue;
301
302         QString error = filename + QLatin1Char(':') + QString::number(m.loc.startLine)
303                         + QLatin1Char(':') + QString::number(m.loc.startColumn) + QLatin1String(": error: ")
304                         + m.message + QLatin1Char('\n');
305
306         int line = 0;
307         if (m.loc.startLine > 0)
308             line = m.loc.startLine - 1;
309
310         const QString textLine = lines.at(line);
311
312         error += textLine + QLatin1Char('\n');
313
314         int column = m.loc.startColumn - 1;
315         if (column < 0)
316             column = 0;
317
318         column = qMin(column, textLine.length());
319
320         for (int i = 0; i < column; ++i) {
321             const QChar ch = textLine.at(i);
322             if (ch.isSpace())
323                 error += ch.unicode();
324             else
325                 error += QLatin1Char(' ');
326         }
327         error += QLatin1String("^\n");
328         errorString += error;
329     }
330     return errorString;
331 }
332
333 bool processComment(const QChar *chars, int length, Comment &comment)
334 {
335     // Try to match the logic of the QtScript parser.
336     if (!length)
337         return comment.isValid();
338     if (*chars == QLatin1Char(':') && chars[1].isSpace()) {
339         comment.extracomment += QString(chars+1, length-1);
340     } else if (*chars == QLatin1Char('=') && chars[1].isSpace()) {
341         comment.msgid = QString(chars+2, length-2).simplified();
342     } else if (*chars == QLatin1Char('~') && chars[1].isSpace()) {
343         QString text = QString(chars+2, length-2).trimmed();
344         int k = text.indexOf(QLatin1Char(' '));
345         if (k > -1)
346             comment.extra.insert(text.left(k), text.mid(k + 1).trimmed());
347     } else if (*chars == QLatin1Char('%') && chars[1].isSpace()) {
348         comment.sourcetext.reserve(comment.sourcetext.length() + length-2);
349         ushort *ptr = (ushort *)comment.sourcetext.data() + comment.sourcetext.length();
350         int p = 2, c;
351         forever {
352             if (p >= length)
353                 break;
354             c = chars[p++].unicode();
355             if (isspace(c))
356                 continue;
357             if (c != '"')
358                 break;
359             forever {
360                 if (p >= length)
361                     break;
362                 c = chars[p++].unicode();
363                 if (c == '"')
364                     break;
365                 if (c == '\\') {
366                     if (p >= length)
367                         break;
368                     c = chars[p++].unicode();
369                     if (c == '\n')
370                         break;
371                     *ptr++ = '\\';
372                 }
373                 *ptr++ = c;
374             }
375         }
376         comment.sourcetext.resize(ptr - (ushort *)comment.sourcetext.data());
377     }
378     return comment.isValid();
379 }
380
381 bool loadQml(Translator &translator, const QString &filename, ConversionData &cd)
382 {
383     cd.m_sourceFileName = filename;
384     QFile file(filename);
385     if (!file.open(QIODevice::ReadOnly)) {
386         cd.appendError(LU::tr("Cannot open %1: %2").arg(filename, file.errorString()));
387         return false;
388     }
389
390     const QString code = QTextStream(&file).readAll();
391
392     Engine driver;
393     Parser parser(&driver);
394
395     NodePool nodePool(filename, &driver);
396     driver.setNodePool(&nodePool);
397
398     Lexer lexer(&driver);
399     lexer.setCode(code, /*line = */ 1);
400     driver.setLexer(&lexer);
401
402     if (parser.parse()) {
403         FindTrCalls trCalls;
404
405         // build up a list of comments that contain translation information.
406         for (int i = 0; i < driver.comments().size(); ++i) {
407             AST::SourceLocation loc = driver.comments().at(i);
408             QString commentStr = code.mid(loc.offset, loc.length);
409
410             if (trCalls.comments.isEmpty() || trCalls.comments.last().lastLine != int(loc.startLine)) {
411                 Comment comment;
412                 comment.lastLine = loc.startLine+1;
413                 if (processComment(commentStr.constData(), commentStr.length(), comment))
414                     trCalls.comments.append(comment);
415             } else {
416                 Comment &lastComment = trCalls.comments.last();
417                 lastComment.lastLine += 1;
418                 processComment(commentStr.constData(), commentStr.length(), lastComment);
419             }
420         }
421
422         //find all tr calls in the code
423         trCalls(&translator, filename, parser.ast());
424     } else {
425         QString error = createErrorString(filename, code, parser);
426         cd.appendError(error);
427         return false;
428     }
429     return true;
430 }
431
432 QT_END_NAMESPACE