feat: RSS support

8 files changed, 373 insertions, 1 deletions

diff --git a/bot/CMakeLists.txt b/bot/CMakeLists.txt
index 261e467..c1e14e1 100644
--- a/bot/CMakeLists.txt
+++ b/bot/CMakeLists.txt
@@ -120,6 +120,22 @@ FetchContent_Declare(
 )
 FetchContent_MakeAvailable(emotespp)
 
+# xml
+FetchContent_Declare(
+  pugixml
+  GIT_REPOSITORY https://github.com/zeux/pugixml.git
+  GIT_TAG v1.15
+)
+FetchContent_MakeAvailable(pugixml)
+
+# fmt
+FetchContent_Declare(
+  fmt
+  GIT_REPOSITORY https://github.com/fmtlib/fmt.git
+  GIT_TAG 12.1.0
+)
+FetchContent_MakeAvailable(fmt)
+
 target_link_libraries(Bot PRIVATE
     ixwebsocket::ixwebsocket
     nlohmann_json::nlohmann_json
@@ -127,5 +143,7 @@ target_link_libraries(Bot PRIVATE
     lua
     sol2::sol2
     emotespp
+    pugixml
+    fmt
 )
 
diff --git a/bot/src/config.cpp b/bot/src/config.cpp
index 9f22edf..4e4b71d 100644
--- a/bot/src/config.cpp
+++ b/bot/src/config.cpp
@@ -65,6 +65,11 @@ namespace bot {
     } else {
       url["randompost"] = sol::lua_nil;
     }
+    if (this->url.rssbridge.has_value()) {
+      url["rssbridge"] = this->url.rssbridge.value();
+    } else {
+      url["rssbridge"] = sol::lua_nil;
+    }
     o["url"] = url;
 
     return o;
@@ -149,6 +154,8 @@ namespace bot {
         url_cfg.paste_service = value;
       } else if (key == "url.randompost") {
         url_cfg.randompost = value;
+      } else if (key == "url.rssbridge") {
+        url_cfg.rssbridge = value;
       }
 
       else if (key == "token.github") {
diff --git a/bot/src/config.hpp b/bot/src/config.hpp
index f6bba7a..139c4f3 100644
--- a/bot/src/config.hpp
+++ b/bot/src/config.hpp
@@ -52,6 +52,7 @@ namespace bot {
       std::optional<std::string> help = std::nullopt;
       std::optional<std::string> paste_service = std::nullopt;
       std::optional<std::string> randompost = std::nullopt;
+      std::optional<std::string> rssbridge = std::nullopt;
   };
 
   struct TokenConfiguration {
diff --git a/bot/src/main.cpp b/bot/src/main.cpp
index a9ed193..9d694e0 100644
--- a/bot/src/main.cpp
+++ b/bot/src/main.cpp
@@ -22,6 +22,7 @@
 #include "irc/message.hpp"
 #include "localization/localization.hpp"
 #include "logger.hpp"
+#include "rss.hpp"
 #include "schemas/stream.hpp"
 #include "stream.hpp"
 #include "timer.hpp"
@@ -193,6 +194,8 @@ int main(int argc, char *argv[]) {
 
 #endif
 
+  bot::RSSListener rss_listener(client, helix_client, cfg);
+
   client.on<bot::irc::MessageType::Privmsg>(
       [&client, &command_loader, &localization, &cfg, &helix_client,
        &kick_api_client](
@@ -214,6 +217,7 @@ int main(int argc, char *argv[]) {
       std::thread(bot::emotes::create_emote_thread, &emote_bundle));
   threads.push_back(std::thread(&bot::api::KickAPIClient::refresh_token_thread,
                                 &kick_api_client));
+  threads.push_back(std::thread(&bot::RSSListener::run, &rss_listener));
 
   for (auto &thread : threads) {
     thread.join();
diff --git a/bot/src/rss.cpp b/bot/src/rss.cpp
new file mode 100644
index 0000000..a52c176
--- /dev/null
+++ b/bot/src/rss.cpp
@@ -0,0 +1,276 @@
+#include "rss.hpp"
+
+#include <fmt/core.h>
+
+#include <algorithm>
+#include <chrono>
+#include <ctime>
+#include <iomanip>
+#include <memory>
+#include <optional>
+#include <pugixml.hpp>
+#include <sstream>
+#include <string>
+#include <thread>
+#include <vector>
+
+#include "cpr/api.h"
+#include "cpr/cprtypes.h"
+#include "cpr/response.h"
+#include "database.hpp"
+#include "fmt/format.h"
+#include "logger.hpp"
+#include "schemas/event.hpp"
+#include "schemas/stream.hpp"
+#include "utils/events.hpp"
+
+namespace bot {
+  sol::table RSSChannel::as_lua_table(std::shared_ptr<sol::state> state) const {
+    sol::table t = state->create_table();
+    t["name"] = this->name;
+    t["url"] = this->url;
+
+    if (this->event.has_value()) {
+      sol::table e = state->create_table();
+      e["name"] = this->event->name;
+      e["type"] = this->event->type;
+      t["event"] = e;
+    } else {
+      t["event"] = sol::lua_nil;
+    }
+
+    sol::table ms = state->create_table();
+
+    for (const RSSMessage &v : this->messages) {
+      sol::table m = state->create_table();
+      m["title"] = v.title;
+      m["message"] = v.message;
+      m["id"] = v.id;
+      m["timestamp"] = v.timestamp;
+      ms.add(m);
+    }
+
+    t["messages"] = ms;
+
+    return t;
+  }
+
+  void RSSListener::run() {
+    if (!this->configuration.url.rssbridge.has_value()) {
+      log::error("RSSListener", "RSS Bridge is not set!");
+      return;
+    }
+
+    while (true) {
+      this->add_channels();
+      this->check_channels();
+      std::this_thread::sleep_for(std::chrono::seconds(30));
+    }
+  }
+
+  bool RSSListener::has_channel(const std::string &url) const {
+    return std::any_of(this->channels.begin(), this->channels.end(),
+                       [&url](const RSSChannel &c) { return c.url == url; });
+  }
+
+  void RSSListener::add_channel(const std::string &url) {
+    if (this->has_channel(url)) return;
+
+    std::optional<RSSChannel> channel = get_rss_channel(url);
+    if (channel.has_value()) {
+      this->channels.push_back(channel.value());
+    }
+  }
+
+  void RSSListener::remove_channel(const std::string &url) {
+    if (!this->has_channel(url)) return;
+
+    std::remove_if(this->channels.begin(), this->channels.end(),
+                   [&url](const RSSChannel &c) { return c.url == url; });
+  }
+
+  void RSSListener::add_channels() {
+    std::unique_ptr<db::BaseDatabase> conn =
+        db::create_connection(this->configuration);
+
+    db::DatabaseRows events = conn->exec(
+        "SELECT event_type, name "
+        "FROM events "
+        "WHERE event_type BETWEEN $1 AND $2",
+        {std::to_string(static_cast<int>(schemas::EventType::RSS)),
+         std::to_string(static_cast<int>(schemas::EventType::TELEGRAM))});
+
+    // adding new events
+    for (db::DatabaseRow event : events) {
+      std::string name = event.at("name");
+      int type = std::stoi(event.at("event_type"));
+      std::string bridge = "";
+      bool useRSSBridge = true;
+
+      switch (type) {
+        case schemas::RSS:
+          bridge = "RSS";
+          useRSSBridge = false;
+          break;
+        case schemas::TWITTER:
+          bridge = "FarsideNitterBridge";
+          break;
+        case schemas::TELEGRAM:
+          bridge = "TelegramBridge";
+          if (name[0] != '@') {
+            name = "%40" + name;
+          }
+          break;
+        default:
+          break;
+      }
+
+      if (bridge.empty()) {
+        log::warn("RSSListener",
+                  "Failed to specify bridge: " + event.at("event_type"));
+        continue;
+      }
+
+      if (std::any_of(this->channels.begin(), this->channels.end(),
+                      [&name, &type](const RSSChannel &c) {
+                        auto e = c.event;
+                        if (!e.has_value()) {
+                          return false;
+                        }
+                        return e->name == name && e->type == type;
+                      })) {
+        continue;
+      }
+
+      std::string url =
+          useRSSBridge
+              ? fmt::format(*this->configuration.url.rssbridge, bridge, name)
+              : name;
+
+      std::optional<RSSChannel> channel = get_rss_channel(url);
+      if (!channel.has_value()) {
+        log::warn("RSSListener", "No RSS feed on " + url);
+        continue;
+      }
+
+      channel->event = {name, type};
+
+      this->channels.push_back(*channel);
+    }
+
+    // removing old events
+    auto channels = this->channels;
+    for (RSSChannel c : channels) {
+      if (!c.event.has_value()) {
+        continue;
+      }
+
+      if (!std::any_of(events.begin(), events.end(),
+                       [&c](const db::DatabaseRow &r) {
+                         return r.at("name") == c.event->name &&
+                                std::stoi(r.at("event_type")) == c.event->type;
+                       })) {
+        this->remove_channel(c.url);
+      }
+    }
+  }
+
+  void RSSListener::check_channels() {
+    for (auto it = this->channels.begin(); it != this->channels.end(); ++it) {
+      if (!it->event.has_value()) {
+        continue;
+      }
+
+      std::optional<RSSChannel> channel = get_rss_channel(it->url);
+      if (!channel.has_value()) {
+        continue;
+      }
+
+      std::vector<RSSMessage> messages;
+      for (auto mit = channel->messages.begin(); mit != channel->messages.end();
+           ++mit) {
+        if (!std::any_of(
+                it->messages.begin(), it->messages.end(),
+                [&mit](const RSSMessage &m) { return m.id == mit->id; })) {
+          messages.push_back(*mit);
+        }
+      }
+
+      if (messages.empty()) {
+        continue;
+      }
+
+      // getting channels
+      std::vector<schemas::Event> events = utils::get_events(
+          db::create_connection(this->configuration), this->helix_client,
+          this->irc_client.get_bot_id(), it->event->type, it->event->name);
+
+      for (const schemas::Event &event : events) {
+        int count = 0;
+
+        for (RSSMessage message : messages) {
+          if (count > 5) break;
+          count++;
+
+          std::string msg = event.message;
+
+          int pos = msg.find("{channel_name}");
+          if (pos != std::string::npos) msg.replace(pos, 14, it->name);
+
+          pos = msg.find("{title}");
+          if (pos != std::string::npos) msg.replace(pos, 7, message.title);
+
+          pos = msg.find("{message}");
+          if (pos != std::string::npos) msg.replace(pos, 9, message.message);
+
+          pos = msg.find("{link}");
+          if (pos != std::string::npos) msg.replace(pos, 6, message.id);
+
+          this->irc_client.say(event.channel_alias_name, msg);
+        }
+      }
+    }
+  }
+
+  std::optional<RSSChannel> get_rss_channel(const std::string &url) {
+    cpr::Response response = cpr::Get(
+        cpr::Url{url},
+        cpr::Header{{"Accept", "application/xml"},
+                    {"User-Agent", "https://github.com/ilotterytea/bot"}});
+
+    if (response.status_code != 200) {
+      return std::nullopt;
+    }
+
+    pugi::xml_document doc;
+    if (!doc.load_string(response.text.c_str())) {
+      return std::nullopt;
+    }
+
+    pugi::xml_node channel = doc.child("rss").child("channel");
+
+    std::string channel_name = channel.child("title").text().as_string();
+    std::string channel_url = channel.child("link").text().as_string();
+
+    std::vector<RSSMessage> messages;
+    for (pugi::xml_node item : channel.children("item")) {
+      // parsing timestamp
+      long timestamp = 0;
+      std::string pubdate = item.child("pubDate").text().as_string();
+      pubdate = pubdate.substr(0, pubdate.size() - 6);
+      std::tm tm = {};
+      std::istringstream ss(pubdate);
+      ss >> std::get_time(&tm, "%a, %d %b %Y %H:%M:%S");
+      if (!ss.fail()) {
+        timestamp = timegm(&tm);
+      }
+
+      messages.push_back({item.child("title").text().as_string(),
+                          item.child("guid").text().as_string(),
+                          item.child("description").text().as_string(),
+                          timestamp});
+    }
+
+    return (RSSChannel){channel_name, channel_url, std::nullopt, messages};
+  }
+}
\ No newline at end of file
diff --git a/bot/src/rss.hpp b/bot/src/rss.hpp
new file mode 100644
index 0000000..cc925ff
--- /dev/null
+++ b/bot/src/rss.hpp
@@ -0,0 +1,57 @@
+#pragma once
+
+#include <optional>
+#include <sol/sol.hpp>
+#include <string>
+#include <vector>
+
+#include "api/twitch/helix_client.hpp"
+#include "config.hpp"
+#include "irc/client.hpp"
+
+namespace bot {
+  struct RSSMessage {
+      std::string title, id, message;
+      long timestamp;
+  };
+
+  struct RSSEvent {
+      std::string name;
+      int type;
+  };
+
+  struct RSSChannel {
+      std::string name, url;
+      std::optional<RSSEvent> event;
+      std::vector<RSSMessage> messages;
+
+      sol::table as_lua_table(std::shared_ptr<sol::state> state) const;
+  };
+
+  class RSSListener {
+    public:
+      RSSListener(irc::Client &irc_client,
+                  api::twitch::HelixClient &helix_client,
+                  Configuration &configuration)
+          : irc_client(irc_client),
+            helix_client(helix_client),
+            configuration(configuration) {};
+      ~RSSListener() = default;
+
+      void run();
+      void add_channel(const std::string &url);
+      void remove_channel(const std::string &url);
+      bool has_channel(const std::string &url) const;
+
+    private:
+      void add_channels();
+      void check_channels();
+
+      std::vector<RSSChannel> channels;
+      irc::Client &irc_client;
+      api::twitch::HelixClient &helix_client;
+      Configuration &configuration;
+  };
+
+  std::optional<RSSChannel> get_rss_channel(const std::string &url);
+}
\ No newline at end of file
diff --git a/bot/src/schemas/stream.cpp b/bot/src/schemas/stream.cpp
index c332fd8..80fc854 100644
--- a/bot/src/schemas/stream.cpp
+++ b/bot/src/schemas/stream.cpp
@@ -36,7 +36,13 @@ namespace bot::schemas {
       return EventType::BTTV_EMOTE_UPDATE;
     }
 #endif
-    else {
+    else if (type == "rss") {
+      return EventType::RSS;
+    } else if (type == "twitter") {
+      return EventType::TWITTER;
+    } else if (type == "telegram") {
+      return EventType::TELEGRAM;
+    } else {
       return EventType::CUSTOM;
     }
   }
diff --git a/bot/src/schemas/stream.hpp b/bot/src/schemas/stream.hpp
index 7e089c7..e0583b4 100644
--- a/bot/src/schemas/stream.hpp
+++ b/bot/src/schemas/stream.hpp
@@ -22,6 +22,9 @@ namespace bot::schemas {
     BTTV_EMOTE_UPDATE,
 #endif
     GITHUB = 40,
+    RSS = 45,
+    TWITTER = 46,
+    TELEGRAM = 47,
     CUSTOM = 99
   };