Merge pull request #5287 from elFarto/master

Implement RSS Smart Filter

Author: glassez

src/base/rss/rss_autodownloader.cpp

@@ -64,6 +64,7 @@ const QString ConfFolderName(QStringLiteral("rss"));
 const QString RulesFileName(QStringLiteral("download_rules.json"));
 
 const QString SettingsKey_ProcessingEnabled(QStringLiteral("RSS/AutoDownloader/EnableProcessing"));
+const QString SettingsKey_SmartEpisodeFilter(QStringLiteral("RSS/AutoDownloader/SmartEpisodeFilter"));
 
 namespace
 {
@@ -95,6 +96,11 @@ using namespace RSS;
 
 QPointer<AutoDownloader> AutoDownloader::m_instance = nullptr;
 
+QString computeSmartFilterRegex(const QStringList &filters)
+{
+    return QString("(?:_|\\b)(?:%1)(?:_|\\b)").arg(filters.join(QString(")|(?:")));
+}
+
 AutoDownloader::AutoDownloader()
     : m_processingEnabled(SettingsStorage::instance()->loadValue(SettingsKey_ProcessingEnabled, false).toBool())
     , m_processingTimer(new QTimer(this))
@@ -123,6 +129,13 @@ AutoDownloader::AutoDownloader()
     connect(BitTorrent::Session::instance(), &BitTorrent::Session::downloadFromUrlFailed
             , this, &AutoDownloader::handleTorrentDownloadFailed);
 
+    // initialise the smart episode regex
+    const QString regex = computeSmartFilterRegex(smartEpisodeFilters());
+    m_smartEpisodeRegex = QRegularExpression(regex,
+                                             QRegularExpression::CaseInsensitiveOption
+                                             | QRegularExpression::ExtendedPatternSyntaxOption
+                                             | QRegularExpression::UseUnicodePropertiesOption);
+
     load();
 
     m_processingTimer->setSingleShot(true);
@@ -266,6 +279,37 @@ void AutoDownloader::importRulesFromLegacyFormat(const QByteArray &data)
         insertRule(AutoDownloadRule::fromLegacyDict(val.toHash()));
 }
 
+QStringList AutoDownloader::smartEpisodeFilters() const
+{
+    const QVariant filtersSetting = SettingsStorage::instance()->loadValue(SettingsKey_SmartEpisodeFilter);
+
+    if (filtersSetting.isNull()) {
+        QStringList filters = {
+            "s(\\d+)e(\\d+)",                       // Format 1: s01e01
+            "(\\d+)x(\\d+)",                        // Format 2: 01x01
+            "(\\d{4}[.\\-]\\d{1,2}[.\\-]\\d{1,2})", // Format 3: 2017.01.01
+            "(\\d{1,2}[.\\-]\\d{1,2}[.\\-]\\d{4})"  // Format 4: 01.01.2017
+        };
+
+        return filters;
+    }
+
+    return filtersSetting.toStringList();
+}
+
+QRegularExpression AutoDownloader::smartEpisodeRegex() const
+{
+    return m_smartEpisodeRegex;
+}
+
+void AutoDownloader::setSmartEpisodeFilters(const QStringList &filters)
+{
+    SettingsStorage::instance()->storeValue(SettingsKey_SmartEpisodeFilter, filters);
+
+    const QString regex = computeSmartFilterRegex(filters);
+    m_smartEpisodeRegex.setPattern(regex);
+}
+
 void AutoDownloader::process()
 {
     if (m_processingQueue.isEmpty()) return; // processing was disabled
@@ -333,6 +377,8 @@ void AutoDownloader::processJob(const QSharedPointer<ProcessingJob> &job)
         }
 
         rule.setLastMatch(articleDate);
+        rule.appendLastComputedEpisode();
+
         m_dirty = true;
         storeDeferred();
 

src/base/rss/rss_autodownloader.h

@@ -35,6 +35,7 @@
 #include <QList>
 #include <QObject>
 #include <QPointer>
+#include <QRegularExpression>
 #include <QSharedPointer>
 
 class QThread;
@@ -80,6 +81,10 @@ namespace RSS
         bool isProcessingEnabled() const;
         void setProcessingEnabled(bool enabled);
 
+        QStringList smartEpisodeFilters() const;
+        void setSmartEpisodeFilters(const QStringList &filters);
+        QRegularExpression smartEpisodeRegex() const;
+
         bool hasRule(const QString &ruleName) const;
         AutoDownloadRule ruleByName(const QString &ruleName) const;
         QList<AutoDownloadRule> rules() const;
@@ -132,5 +137,6 @@ namespace RSS
         QHash<QString, QSharedPointer<ProcessingJob>> m_waitingJobs;
         bool m_dirty = false;
         QBasicTimer m_savingTimer;
+        QRegularExpression m_smartEpisodeRegex;
     };
 }

src/base/rss/rss_autodownloadrule.cpp

@@ -34,7 +34,6 @@
 #include <QHash>
 #include <QJsonArray>
 #include <QJsonObject>
-#include <QRegExp>
 #include <QRegularExpression>
 #include <QSharedData>
 #include <QString>
@@ -46,6 +45,7 @@
 #include "../utils/string.h"
 #include "rss_feed.h"
 #include "rss_article.h"
+#include "rss_autodownloader.h"
 
 namespace
 {
@@ -100,6 +100,8 @@ const QString Str_AssignedCategory(QStringLiteral("assignedCategory"));
 const QString Str_LastMatch(QStringLiteral("lastMatch"));
 const QString Str_IgnoreDays(QStringLiteral("ignoreDays"));
 const QString Str_AddPaused(QStringLiteral("addPaused"));
+const QString Str_SmartFilter(QStringLiteral("smartFilter"));
+const QString Str_PreviouslyMatched(QStringLiteral("previouslyMatchedEpisodes"));
 
 namespace RSS
 {
@@ -120,6 +122,10 @@ namespace RSS
         QString category;
         TriStateBool addPaused = TriStateBool::Undefined;
 
+        bool smartFilter = false;
+        QStringList previouslyMatchedEpisodes;
+
+        mutable QString lastComputedEpisode;
         mutable QHash<QString, QRegularExpression> cachedRegexes;
 
         bool operator==(const AutoDownloadRuleData &other) const
@@ -135,13 +141,38 @@ namespace RSS
                     && (lastMatch == other.lastMatch)
                     && (savePath == other.savePath)
                     && (category == other.category)
-                    && (addPaused == other.addPaused);
+                    && (addPaused == other.addPaused)
+                    && (smartFilter == other.smartFilter);
         }
     };
 }
 
 using namespace RSS;
 
+QString computeEpisodeName(const QString &article)
+{
+    const QRegularExpression episodeRegex = AutoDownloader::instance()->smartEpisodeRegex();
+    const QRegularExpressionMatch match = episodeRegex.match(article);
+
+    // See if we can extract an season/episode number or date from the title
+    if (!match.hasMatch())
+        return QString();
+
+    QStringList ret;
+    for (int i = 1; i <= match.lastCapturedIndex(); ++i) {
+        QString cap = match.captured(i);
+
+        if (cap.isEmpty())
+            continue;
+
+        bool isInt = false;
+        int x = cap.toInt(&isInt);
+
+        ret.append(isInt ? QString::number(x) : cap);
+    }
+    return ret.join('x');
+}
+
 AutoDownloadRule::AutoDownloadRule(const QString &name)
     : m_dataPtr(new AutoDownloadRuleData)
 {
@@ -197,6 +228,9 @@ bool AutoDownloadRule::matches(const QString &articleTitle, const QString &expre
 
 bool AutoDownloadRule::matches(const QString &articleTitle) const
 {
+    // Reset the lastComputedEpisode, we don't want to leak it between matches
+    m_dataPtr->lastComputedEpisode.clear();
+
     if (!m_dataPtr->mustContain.empty()) {
         bool logged = false;
         bool foundMustContain = false;
@@ -334,6 +368,20 @@ bool AutoDownloadRule::matches(const QString &articleTitle) const
         return false;
     }
 
+    if (useSmartFilter()) {
+        // now see if this episode has been downloaded before
+        const QString episodeStr = computeEpisodeName(articleTitle);
+
+        if (!episodeStr.isEmpty()) {
+            bool previouslyMatched = m_dataPtr->previouslyMatchedEpisodes.contains(episodeStr);
+            bool isRepack = articleTitle.contains("REPACK", Qt::CaseInsensitive) || articleTitle.contains("PROPER", Qt::CaseInsensitive);
+            if (previouslyMatched && !isRepack)
+                return false;
+
+            m_dataPtr->lastComputedEpisode = episodeStr;
+        }
+    }
+
 //    qDebug() << "Matched article:" << articleTitle;
     return true;
 }
@@ -367,7 +415,9 @@ QJsonObject AutoDownloadRule::toJsonObject() const
         , {Str_AssignedCategory, assignedCategory()}
         , {Str_LastMatch, lastMatch().toString(Qt::RFC2822Date)}
         , {Str_IgnoreDays, ignoreDays()}
-        , {Str_AddPaused, triStateBoolToJsonValue(addPaused())}};
+        , {Str_AddPaused, triStateBoolToJsonValue(addPaused())}
+        , {Str_SmartFilter, useSmartFilter()}
+        , {Str_PreviouslyMatched, QJsonArray::fromStringList(previouslyMatchedEpisodes())}};
 }
 
 AutoDownloadRule AutoDownloadRule::fromJsonObject(const QJsonObject &jsonObj, const QString &name)
@@ -384,6 +434,7 @@ AutoDownloadRule AutoDownloadRule::fromJsonObject(const QJsonObject &jsonObj, co
     rule.setAddPaused(jsonValueToTriStateBool(jsonObj.value(Str_AddPaused)));
     rule.setLastMatch(QDateTime::fromString(jsonObj.value(Str_LastMatch).toString(), Qt::RFC2822Date));
     rule.setIgnoreDays(jsonObj.value(Str_IgnoreDays).toInt());
+    rule.setUseSmartFilter(jsonObj.value(Str_SmartFilter).toBool(false));
 
     const QJsonValue feedsVal = jsonObj.value(Str_AffectedFeeds);
     QStringList feedURLs;
@@ -393,6 +444,17 @@ AutoDownloadRule AutoDownloadRule::fromJsonObject(const QJsonObject &jsonObj, co
         feedURLs << urlVal.toString();
     rule.setFeedURLs(feedURLs);
 
+    const QJsonValue previouslyMatchedVal = jsonObj.value(Str_PreviouslyMatched);
+    QStringList previouslyMatched;
+    if (previouslyMatchedVal.isString()) {
+        previouslyMatched << previouslyMatchedVal.toString();
+    }
+    else {
+        foreach (const QJsonValue &val, previouslyMatchedVal.toArray())
+            previouslyMatched << val.toString();
+    }
+    rule.setPreviouslyMatchedEpisodes(previouslyMatched);
+
     return rule;
 }
 
@@ -549,6 +611,16 @@ QString AutoDownloadRule::mustNotContain() const
     return m_dataPtr->mustNotContain.join("|");
 }
 
+bool AutoDownloadRule::useSmartFilter() const
+{
+    return m_dataPtr->smartFilter;
+}
+
+void AutoDownloadRule::setUseSmartFilter(bool enabled)
+{
+    m_dataPtr->smartFilter = enabled;
+}
+
 bool AutoDownloadRule::useRegex() const
 {
     return m_dataPtr->useRegex;
@@ -560,6 +632,25 @@ void AutoDownloadRule::setUseRegex(bool enabled)
     m_dataPtr->cachedRegexes.clear();
 }
 
+QStringList AutoDownloadRule::previouslyMatchedEpisodes() const
+{
+    return m_dataPtr->previouslyMatchedEpisodes;
+}
+
+void AutoDownloadRule::setPreviouslyMatchedEpisodes(const QStringList &previouslyMatchedEpisodes)
+{
+    m_dataPtr->previouslyMatchedEpisodes = previouslyMatchedEpisodes;
+}
+
+void AutoDownloadRule::appendLastComputedEpisode()
+{
+    if (!m_dataPtr->lastComputedEpisode.isEmpty()) {
+        // TODO: probably need to add a marker for PROPER/REPACK to avoid duplicate downloads
+        m_dataPtr->previouslyMatchedEpisodes.append(m_dataPtr->lastComputedEpisode);
+        m_dataPtr->lastComputedEpisode.clear();
+    }
+}
+
 QString AutoDownloadRule::episodeFilter() const
 {
     return m_dataPtr->episodeFilter;

src/base/rss/rss_autodownloadrule.h

@@ -66,9 +66,15 @@ namespace RSS
         void setLastMatch(const QDateTime &lastMatch);
         bool useRegex() const;
         void setUseRegex(bool enabled);
+        bool useSmartFilter() const;
+        void setUseSmartFilter(bool enabled);
         QString episodeFilter() const;
         void setEpisodeFilter(const QString &e);
 
+        void appendLastComputedEpisode();
+        QStringList previouslyMatchedEpisodes() const;
+        void setPreviouslyMatchedEpisodes(const QStringList &previouslyMatchedEpisodes);
+
         QString savePath() const;
         void setSavePath(const QString &savePath);
         TriStateBool addPaused() const;

src/gui/optionsdlg.cpp

@@ -377,6 +377,7 @@ OptionsDialog::OptionsDialog(QWidget *parent)
     // RSS tab
     connect(m_ui->checkRSSEnable, &QCheckBox::toggled, this, &OptionsDialog::enableApplyButton);
     connect(m_ui->checkRSSAutoDownloaderEnable, &QCheckBox::toggled, this, &OptionsDialog::enableApplyButton);
+    connect(m_ui->textSmartEpisodeFilters, &QPlainTextEdit::textChanged, this, &OptionsDialog::enableApplyButton);
     connect(m_ui->spinRSSRefreshInterval, qSpinBoxValueChanged, this, &OptionsDialog::enableApplyButton);
     connect(m_ui->spinRSSMaxArticlesPerFeed, qSpinBoxValueChanged, this, &OptionsDialog::enableApplyButton);
     connect(m_ui->btnEditRules, &QPushButton::clicked, [this]() { AutomatedRssDownloader(this).exec(); });
@@ -553,6 +554,7 @@ void OptionsDialog::saveOptions()
     RSS::Session::instance()->setMaxArticlesPerFeed(m_ui->spinRSSMaxArticlesPerFeed->value());
     RSS::Session::instance()->setProcessingEnabled(m_ui->checkRSSEnable->isChecked());
     RSS::AutoDownloader::instance()->setProcessingEnabled(m_ui->checkRSSAutoDownloaderEnable->isChecked());
+    RSS::AutoDownloader::instance()->setSmartEpisodeFilters(m_ui->textSmartEpisodeFilters->toPlainText().split('\n', QString::SplitBehavior::SkipEmptyParts));
 
     auto session = BitTorrent::Session::instance();
 
@@ -780,6 +782,8 @@ void OptionsDialog::loadOptions()
 
     m_ui->checkRSSEnable->setChecked(RSS::Session::instance()->isProcessingEnabled());
     m_ui->checkRSSAutoDownloaderEnable->setChecked(RSS::AutoDownloader::instance()->isProcessingEnabled());
+    m_ui->textSmartEpisodeFilters->setPlainText(RSS::AutoDownloader::instance()->smartEpisodeFilters().join('\n'));
+
     m_ui->spinRSSRefreshInterval->setValue(RSS::Session::instance()->refreshInterval());
     m_ui->spinRSSMaxArticlesPerFeed->setValue(RSS::Session::instance()->maxArticlesPerFeed());
 

src/gui/optionsdlg.ui

@@ -2699,6 +2699,18 @@
               </layout>
              </widget>
             </item>
+            <item>
+             <widget class="QGroupBox" name="groupRSSSmartEpisodeFilter">
+              <property name="title">
+               <string>RSS Smart Episode Filters</string>
+              </property>
+              <layout class="QVBoxLayout" name="verticalLayout_31">
+               <item>
+                <widget class="QPlainTextEdit" name="textSmartEpisodeFilters"/>
+               </item>
+              </layout>
+             </widget>
+            </item>
             <item>
              <spacer name="verticalSpacer_5">
               <property name="orientation">
@@ -2707,7 +2719,7 @@
               <property name="sizeHint" stdset="0">
                <size>
                 <width>20</width>
-                <height>267</height>
+                <height>200</height>
                </size>
               </property>
              </spacer>