Use QTokenizer to properly parse netrc

Addresses #4691

src/3rdparty/qtokenizer/qtokenizer.h

@@ -0,0 +1,264 @@
+/****************************************************************************
+**
+** Copyright (C) 2014 Daniel Molkentin <daniel@molkentin.de>
+** Contact: http://www.qt-project.org/legal
+**
+** This file is part of the QtNetwork module of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and Digia.  For licensing terms and
+** conditions see http://qt.digia.com/licensing.  For further information
+** use the contact form at http://qt.digia.com/contact-us.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL included in the
+** packaging of this file.  Please review the following information to
+** ensure the GNU Lesser General Public License version 2.1 requirements
+** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Digia gives you certain additional
+** rights.  These rights are described in the Digia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3.0 as published by the Free Software
+** Foundation and appearing in the file LICENSE.GPL included in the
+** packaging of this file.  Please review the following information to
+** ensure the GNU General Public License version 3.0 requirements will be
+** met: http://www.gnu.org/copyleft/gpl.html.
+**
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#ifndef TOKENIZER_H
+#define TOKENIZER_H
+
+#include <QString>
+#include <QByteArray>
+#include <QSharedPointer>
+
+QT_BEGIN_NAMESPACE
+
+template <class T, class const_iterator>
+struct QTokenizerPrivate {
+    typedef typename T::value_type char_type;
+
+    struct State {
+        bool inQuote;
+        bool inEscape;
+        char_type quoteChar;
+        State() : inQuote(false), inEscape(false), quoteChar('\0') {}
+    };
+
+    QTokenizerPrivate(const T& _string, const T& _delims) :
+        string(_string)
+      , begin(string.begin())
+      , end(string.end())
+      , tokenBegin(end)
+      , tokenEnd(begin)
+      , delimiters(_delims)
+      , isDelim(false)
+      , returnDelimiters(false)
+      , returnQuotes(false)
+    {
+    }
+
+    bool isDelimiter(char_type c) const {
+        return delimiters.contains(c);
+    }
+
+    bool isQuote(char_type c) const {
+        return quotes.contains(c);
+    }
+
+    // Returns true if a delimiter was not hit
+    bool nextChar(State* state, char_type c) {
+        if (state->inQuote) {
+            if (state->inEscape) {
+                state->inEscape = false;
+            } else if (c == '\\') {
+                state->inEscape = true;
+            } else if (c == state->quoteChar) {
+                state->inQuote = false;
+            }
+        } else {
+            if (isDelimiter(c))
+                return false;
+            state->inQuote = isQuote(state->quoteChar = c);
+        }
+        return true;
+    }
+
+    T string;
+    // ### copies begin and end for performance, premature optimization?
+    const_iterator begin;
+    const_iterator end;
+    const_iterator tokenBegin;
+    const_iterator tokenEnd;
+    T delimiters;
+    T quotes;
+    bool isDelim;
+    bool returnDelimiters;
+    bool returnQuotes;
+};
+
+template <class T, class const_iterator>
+class QTokenizer {
+public:
+    typedef typename T::value_type char_type;
+
+    /*!
+       \class QTokenizer
+       \inmodule QtNetwork
+       \brief QTokenizer tokenizes Strings on QString, QByteArray,
+              std::string or std::wstring
+
+       Example Usage:
+
+       \code
+         QString str = ...;
+         QByteArrayTokenizer tokenizer(str, "; ");
+         tokenizer.setQuoteCharacters("\"'");
+         tokenizer.setReturnDelimiters(true);
+         while (tokenizer.hasNext()) {
+           QByteArray token = tokenizer.next();
+           bool isDelimiter = tokenizer.isDelimiter();
+           ...
+         }
+       \endcode
+
+       \param string The string to tokenize
+       \param delimiters A string containing delimiters
+
+       \sa QStringTokenizer, QByteArrayTokenizer, StringTokenizer, WStringTokenizer
+     */
+    QTokenizer(const T& string, const T& delimiters) {
+        d.reset(new QTokenizerPrivate<T, const_iterator>(string, delimiters));
+    }
+
+    /*!
+       Whether or not to return delimiters as tokens
+       \see setQuoteCharacters
+     */
+    void setReturnDelimiters(bool enable) { d->returnDelimiters = enable; }
+
+
+    /*!
+       Sets characters that are considered to start and end quotes.
+
+       When between two characters considered a quote, delimiters will
+       be ignored.
+
+       When between quotes, blackslash characters will cause the QTokenizer
+       to skip the next character.
+
+       \param quotes Characters that delimit quotes.
+     */
+    void setQuoteCharacters(const T& quotes) { d->quotes = quotes; }
+
+
+    /*!
+       Whether or not to return delimiters as tokens
+       \see setQuoteCharacters
+     */
+    void setReturnQuoteCharacters(bool enable) { d->returnQuotes = enable; }
+
+
+    /*!
+       Retrieve next token.
+
+       Returns true if there are more tokens, false otherwise.
+
+       \sa next()
+     */
+    bool hasNext()
+    {
+        typename QTokenizerPrivate<T, const_iterator>::State state;
+        d->isDelim = false;
+        for (;;) {
+            d->tokenBegin = d->tokenEnd;
+            if (d->tokenEnd == d->end)
+                return false;
+            d->tokenEnd++;
+            if (d->nextChar(&state, *d->tokenBegin))
+                break;
+            if (d->returnDelimiters) {
+                d->isDelim = true;
+                return true;
+            }
+        }
+        while (d->tokenEnd != d->end && d->nextChar(&state, *d->tokenEnd)) {
+            d->tokenEnd++;
+        }
+        return true;
+    }
+
+    /*!
+       Resets the tokenizer to the starting position.
+     */
+    void reset() {
+        d->tokenEnd = d->begin;
+    }
+
+    /*!
+       Returns true if the current token is a delimiter,
+       if one more more delimiting characters have been set.
+     */
+    bool isDelimiter() const { return d->isDelim; }
+
+    /*!
+       Returns the current token.
+
+       Use \c hasNext() to fetch the next token.
+     */
+    T next() const {
+        int len = d->tokenEnd-d->tokenBegin;
+        const_iterator tmpStart = d->tokenBegin;
+        if (!d->returnQuotes && len > 1 && d->isQuote(*d->tokenBegin)) {
+            tmpStart++;
+            len -= 2;
+        }
+        return T(tmpStart, len);
+    }
+
+private:
+    friend class QStringTokenizer;
+    QSharedPointer<QTokenizerPrivate<T, const_iterator> > d;
+};
+
+class QStringTokenizer : public QTokenizer<QString, QString::const_iterator> {
+public:
+    QStringTokenizer(const QString &string, const QString &delim) :
+        QTokenizer<QString, QString::const_iterator>(string, delim) {}
+    /**
+     * @brief Like \see next(), but returns a lightweight string reference
+     * @return A reference to the token within the string
+     */
+    QStringRef stringRef() {
+        int begin = d->tokenBegin-d->begin;
+        int end = d->tokenEnd-d->tokenBegin;
+        if (!d->returnQuotes && d->isQuote(*d->tokenBegin)) {
+            begin++;
+            end -= 2;
+        }
+        return QStringRef(&d->string, begin, end);
+    }
+};
+
+typedef QTokenizer<QByteArray, QByteArray::const_iterator> QByteArrayTokenizer;
+typedef QTokenizer<std::string, std::string::const_iterator> StringTokenizer;
+typedef QTokenizer<std::wstring, std::wstring::const_iterator> WStringTokenizer;
+
+QT_END_NAMESPACE
+
+#endif // TOKENIZER_H
+

src/3rdparty/qtokenizer/qtokenizer.pro

@@ -0,0 +1,2 @@
+TEMPLATE = subdirs
+SUBDIRS = test

src/3rdparty/qtokenizer/test/test.pro

@@ -0,0 +1,8 @@
+TEMPLATE = app
+QT += testlib
+CONFIG += testlib
+TARGET = test
+INCLUDEPATH += . ..
+
+# Input
+SOURCES += tst_qtokenizer.cpp

src/3rdparty/qtokenizer/test/tst_qtokenizer.cpp

@@ -0,0 +1,139 @@
+#include <QtTest>
+
+#include "qtokenizer.h"
+
+namespace {
+  const QString simple     = QLatin1String("A simple tokenizer test");
+  const QString quoted     = QLatin1String("\"Wait for me!\" he shouted");
+}
+
+class TestTokenizer : public QObject
+{
+  Q_OBJECT
+private slots:
+  void tokenizeQStringSimple() {
+    QStringTokenizer tokenizer(simple, " ");
+
+    QCOMPARE(tokenizer.hasNext(), true);
+    QCOMPARE(tokenizer.next(), QLatin1String("A"));
+
+    QCOMPARE(tokenizer.hasNext(), true);
+    QCOMPARE(tokenizer.next(), QLatin1String("simple"));
+
+    QCOMPARE(tokenizer.hasNext(), true);
+    QCOMPARE(tokenizer.next(), QLatin1String("tokenizer"));
+
+    QCOMPARE(tokenizer.hasNext(), true);
+    QCOMPARE(tokenizer.next(), QLatin1String("test"));
+
+    QCOMPARE(tokenizer.hasNext(), false);
+  }
+
+  void tokenizeQStringSimpleRef() {
+    QStringTokenizer tokenizer(simple, " ");
+
+    QCOMPARE(tokenizer.hasNext(), true);
+    QVERIFY(tokenizer.stringRef() == QLatin1String("A"));
+
+    QCOMPARE(tokenizer.hasNext(), true);
+    QVERIFY(tokenizer.stringRef() == QLatin1String("simple"));
+
+    QCOMPARE(tokenizer.hasNext(), true);
+    QVERIFY(tokenizer.stringRef() == QLatin1String("tokenizer"));
+
+    QCOMPARE(tokenizer.hasNext(), true);
+    QVERIFY(tokenizer.stringRef() == QLatin1String("test"));
+
+    QCOMPARE(tokenizer.hasNext(), false);
+  }
+
+  void tokenizeQStringQuoted() {
+    const QString multiquote(QLatin1String("\"'Billy - the Kid' is dead!\""));
+    QStringTokenizer tokenizer(multiquote, " -");
+    tokenizer.setQuoteCharacters("\"");
+    tokenizer.setReturnQuoteCharacters(true);
+
+    QCOMPARE(tokenizer.hasNext(), true);
+    QCOMPARE(tokenizer.next(), QLatin1String("\"'Billy - the Kid' is dead!\""));
+
+    QCOMPARE(tokenizer.hasNext(), false);
+  }
+
+  void tokenizeQStringSkipQuotes() {
+    const QString multiquote(QLatin1String("\"'Billy - the Kid' is dead!\""));
+    QStringTokenizer tokenizer(multiquote, " ");
+    tokenizer.setQuoteCharacters("\"");
+    tokenizer.setReturnQuoteCharacters(false);
+
+    QCOMPARE(tokenizer.hasNext(), true);
+    QCOMPARE(tokenizer.next(), QLatin1String("'Billy - the Kid' is dead!"));
+    QCOMPARE(tokenizer.stringRef().toString(), QLatin1String("'Billy - the Kid' is dead!"));
+
+    QCOMPARE(tokenizer.hasNext(), false);
+  }
+
+
+  void tokenizeQStringWithDelims() {
+    const QString delims(QLatin1String("I;Insist,On/a-Delimiter"));
+    QStringTokenizer tokenizer(delims, ";,/-");
+    tokenizer.setReturnDelimiters(true);
+
+    QCOMPARE(tokenizer.hasNext(), true);
+    QCOMPARE(tokenizer.isDelimiter(), false);
+
+    QCOMPARE(tokenizer.hasNext(), true);
+    QCOMPARE(tokenizer.isDelimiter(), true);
+
+    QCOMPARE(tokenizer.hasNext(), true);
+    QCOMPARE(tokenizer.isDelimiter(), false);
+
+    QCOMPARE(tokenizer.hasNext(), true);
+    QCOMPARE(tokenizer.isDelimiter(), true);
+
+    QCOMPARE(tokenizer.hasNext(), true);
+    QCOMPARE(tokenizer.isDelimiter(), false);
+
+    QCOMPARE(tokenizer.hasNext(), true);
+    QCOMPARE(tokenizer.isDelimiter(), true);
+
+    QCOMPARE(tokenizer.hasNext(), true);
+    QCOMPARE(tokenizer.isDelimiter(), false);
+
+    QCOMPARE(tokenizer.hasNext(), true);
+    QCOMPARE(tokenizer.isDelimiter(), true);
+
+    QCOMPARE(tokenizer.hasNext(), true);
+    QCOMPARE(tokenizer.isDelimiter(), false);
+
+    QCOMPARE(tokenizer.hasNext(), false);
+  }
+
+  void resetTokenizer() {
+    for (int i = 0; i < 2; i++) {
+      QStringTokenizer tokenizer(simple, " ");
+
+      QCOMPARE(tokenizer.hasNext(), true);
+      QCOMPARE(tokenizer.next(), QLatin1String("A"));
+
+      QCOMPARE(tokenizer.hasNext(), true);
+      QCOMPARE(tokenizer.next(), QLatin1String("simple"));
+
+      QCOMPARE(tokenizer.hasNext(), true);
+      QCOMPARE(tokenizer.next(), QLatin1String("tokenizer"));
+
+      QCOMPARE(tokenizer.hasNext(), true);
+      QCOMPARE(tokenizer.next(), QLatin1String("test"));
+
+      QCOMPARE(tokenizer.hasNext(), false);
+
+      tokenizer.reset();
+    }
+  }
+
+  // ### QByteArray, other types
+};
+
+QTEST_APPLESS_MAIN(TestTokenizer)
+
+#include "tst_qtokenizer.moc"
+

src/cmd/CMakeLists.txt

@@ -17,6 +17,9 @@ include_directories(${CMAKE_SOURCE_DIR}/csync/src
                     ${CMAKE_BINARY_DIR}/csync/src
                    )
 
+# Need tokenizer for netrc parser
+include_directories(${CMAKE_SOURCE_DIR}/src/3rdparty/qtokenizer)
+
 if(NOT BUILD_LIBRARIES_ONLY)
    add_executable(${cmd_NAME}  ${cmd_SRC})
 	qt5_use_modules(${cmd_NAME} Network Sql)

src/cmd/netrcparser.cpp

@@ -16,6 +16,10 @@
 #include <QFile>
 #include <QTextStream>
 
+#include <qtokenizer.h>
+
+#include <QDebug>
+
 #include "netrcparser.h"
 
 namespace OCC {
@@ -28,11 +32,11 @@ QString passwordKeyword = QLatin1String("password");
 
 }
 
-NetrcParser::NetrcParser(const QString &fileName)
-    : _fileName(fileName)
+NetrcParser::NetrcParser(const QString &file)
 {
-    if (_fileName.isEmpty()) {
-       _fileName = QDir::homePath()+QLatin1String("/.netrc");
+    _netrcLocation = file;
+    if (_netrcLocation.isEmpty()) {
+       _netrcLocation = QDir::homePath()+QLatin1String("/.netrc");
     }
 }
 
@@ -49,29 +53,39 @@ void NetrcParser::tryAddEntryAndClear(QString& machine, LoginPair& pair, bool& i
 
 bool NetrcParser::parse()
 {
-    QFile netrc(_fileName);
+    QFile netrc(_netrcLocation);
     if (!netrc.open(QIODevice::ReadOnly)) {
         return false;
     }
+    QString content = netrc.readAll();
+
+    QStringTokenizer tokenizer(content, " \n\t");
+    tokenizer.setQuoteCharacters("\"'");
 
-    QTextStream ts(&netrc);
     LoginPair pair;
     QString machine;
     bool isDefault = false;
-    while (!ts.atEnd()) {
-        QString next;
-        ts >> next;
-        if (next == defaultKeyword) {
+    while (tokenizer.hasNext()) {
+        QString key = tokenizer.next();
+        if (key == defaultKeyword) {
             tryAddEntryAndClear(machine, pair, isDefault);
             isDefault = true;
+            continue; // don't read a value
         }
-        if (next == machineKeyword) {
+
+        if (!tokenizer.hasNext()) {
+            qDebug() << "error fetching value for" << key;
+            return false;
+        }
+        QString value = tokenizer.next();
+
+        if (key == machineKeyword) {
             tryAddEntryAndClear(machine, pair, isDefault);
-            ts >> machine;
-        } else if (next == loginKeyword) {
-            ts >> pair.first;
-        } else if (next == passwordKeyword) {
-            ts >> pair.second;
+            machine = value;
+        } else if (key == loginKeyword) {
+            pair.first = value;
+        } else if (key == passwordKeyword) {
+            pair.second = value;
         } // ignore unsupported tokens
 
     }

src/cmd/netrcparser.h

@@ -29,7 +29,7 @@ class NetrcParser
 public:
     typedef QPair<QString, QString> LoginPair;
 
-    NetrcParser(const QString &fileName = QString::null);
+    NetrcParser(const QString &file = QString());
     bool parse();
     LoginPair find(const QString &machine);
 
@@ -37,7 +37,7 @@ private:
     void tryAddEntryAndClear(QString &machine, LoginPair &pair, bool &isDefault);
     QHash<QString, LoginPair> _entries;
     LoginPair _default;
-    QString _fileName;
+    QString _netrcLocation;
 };
 
 } // namespace OCC

test/CMakeLists.txt

@@ -1,6 +1,7 @@
 include_directories(${CMAKE_BINARY_DIR}/csync ${CMAKE_BINARY_DIR}/csync/src ${CMAKE_BINARY_DIR}/src)
 include_directories(${CMAKE_SOURCE_DIR}/csync/src/)
 include_directories(${CMAKE_SOURCE_DIR}/csync/src/std ${CMAKE_SOURCE_DIR}/src)
+include_directories(${CMAKE_SOURCE_DIR}/src/3rdparty/qtokenizer)
 
 include(QtVersionAbstraction)
 setup_qt()

test/testnetrcparser.cpp

@@ -29,6 +29,7 @@ private slots:
        netrc.write("machine foo login bar password baz\n");
        netrc.write("machine broken login bar2 dontbelonghere password baz2 extratokens dontcare andanother\n");
        netrc.write("machine\nfunnysplit\tlogin bar3 password baz3\n");
+       netrc.write("machine frob login \"user with spaces\" password 'space pwd'\n");
        QFile netrcWithDefault(testfileWithDefaultC);
        QVERIFY(netrcWithDefault.open(QIODevice::WriteOnly));
        netrcWithDefault.write("machine foo login bar password baz\n");
@@ -47,8 +48,9 @@ private slots:
        NetrcParser parser(testfileC);
        QVERIFY(parser.parse());
        QCOMPARE(parser.find("foo"), qMakePair(QString("bar"), QString("baz")));
-       QCOMPARE(parser.find("broken"), qMakePair(QString("bar2"), QString("baz2")));
+       QCOMPARE(parser.find("broken"), qMakePair(QString("bar2"), QString()));
        QCOMPARE(parser.find("funnysplit"), qMakePair(QString("bar3"), QString("baz3")));
+       QCOMPARE(parser.find("frob"), qMakePair(QString("user with spaces"), QString("space pwd")));
     }
 
     void testEmptyNetrc() {