utils.hpp

Go to the documentation of this file.

#pragma once
 
#include <algorithm>
#include <string>
#include <sstream>
#include <pugixml.hpp>
#include <vector>
#include <chrono>
#include <fmt/chrono.h>
 
#include "extraction_param.hpp"
 
namespace SynDomUtils {
 
struct xml_string_writer: xml_writer {
    std::string result;
    virtual void write(const void* data, size_t size) {
        result.append(static_cast<const char*>(data), size);
    }
};
 
const std::string ATOM_LINK_TAGS[] = {
    "link", "atom:link", "atom10:link"
};
 
inline bool __trim_filter(unsigned char ch) {
    return !std::isspace(ch);
}
 
inline void ltrim(std::string &s) {
    s.erase(s.begin(), std::find_if(s.begin(), s.end(), __trim_filter));
}
 
inline void rtrim(std::string &s) {
    s.erase(
        std::find_if(s.rbegin(), s.rend(), __trim_filter).base(), s.end()
    );
}
 
inline void trim(std::string &s) { ltrim(s); rtrim(s); }
 
inline void lower(std::string &s) {
    std::transform(s.begin(), s.end(), s.begin(), ::tolower);
}
 
inline bool str_has_prefix(std::string s, std::string prefix) {
    return (s.rfind(prefix, 0) == 0);
}
 
inline std::vector<std::string> split(std::string s, char delim) {
    std::vector<std::string> result;
    std::stringstream ss(s);
    std::string item;
    while (getline (ss, item, delim)) {
        result.push_back(item);
    }
    return result;
}
 
inline std::string current_time() {
    auto now = std::chrono::system_clock::now();
    std::string res = fmt::format("{:%Y-%m-%dT%H:%M:%S}", now);
    return res;
}
 
inline bool is_url(std::string s) {
    return !s.empty() && (
        str_has_prefix(s, "https://") ||
        str_has_prefix(s, "http://")
    );
}
 
inline std::string extract_from_node(
        pugi::xml_node node, std::vector<ExtractionParam> params
) {
    std::string res = "";
    for (auto param: params) {
        auto child = node;
        for (auto tag: param.tags) {
            if (!child) break;
            child = child.child(tag.c_str());
        }
        if (!child) continue;
 
        if (param.type == ExtractionParam::ParamType::CHILD) {
            res = child.text().as_string();
            // if the child content is not CDATA or text, then it's probably
            // xhtml, so we grab that with the writer
            if (
                    child && res.empty() &&
                    child.first_child().type() != node_cdata &&
                    child.first_child()
            ) {
                xml_node child_cpy = child;
                child_cpy.set_name("article");
                xml_string_writer writer;
                child_cpy.print(writer, "");
                if (writer.result != "<article />") {
                    res = writer.result;
                }
            }
        }
        else {
            res = child.attribute(param.attribute.c_str()).value();
        }
 
        if (!res.empty()) {
            trim(res);
            return res;
        }
    }
    return "";
}
 
inline std::string extract_link(
        pugi::xml_node node,
        std::vector<std::string> rels,
        std::vector<std::string> types,
        bool opt_rel=false, bool opt_type=false
) {
    std::string res = "";
    std::string attr_rel, attr_type;
    for (auto link_tag: ATOM_LINK_TAGS) {
        pugi::xml_node link_node = node.child(link_tag.c_str());
        while(link_node) {
            attr_rel = link_node.attribute("rel").value();
            attr_type = link_node.attribute("type").value();
            if (
                    ((opt_rel && attr_rel.empty()) || std::find(
                        rels.begin(), rels.end(), attr_rel
                    ) != rels.end()) &&
                    ((opt_type && attr_type.empty()) || std::find(
                        types.begin(), types.end(), attr_type
                    ) != types.end())
            ) {
                res = link_node.attribute("href").value();
                if (!res.empty()) return res;
            }
            link_node = link_node.next_sibling(link_tag.c_str());
        }
    }
 
    return "";
}
 
};