Дедупликатор документов

RobertDeMilo

Dec 13th, 2023 (edited)

Never

Add comment

Not a member of Pastebin yet? Sign Up, it unlocks many cool features!

C++ 20.24 KB | None | 0 0

raw download clone embed print report

main
#include <algorithm>
#include <cmath>
#include <iostream>
#include <map>
#include <set>
#include <stdexcept>
#include <string>
#include <utility>
#include <vector>
#include <deque>
#include "search_server.h"
#include "string_processing.h"
#include "document.h"
#include "paginator.h"
#include "request_queue.h"
#include "read_input_functions.h"
#include "remove_duplicates.h"
using namespace std;
template <typename Container>
auto Paginate(const Container& c, size_t page_size) {
return Paginator(begin(c), end(c), page_size);
}
int main() {
SearchServer search_server("and with"s);
//...
}
##############################################################################################################
document.h
#pragma once
struct Document {
Document() = default;
Document(int id, double relevance, int rating)
: id(id)
, relevance(relevance)
, rating(rating)
{
}
int id = 0;
double relevance = 0.0;
int rating = 0;
};
enum class DocumentStatus {
ACTUAL,
IRRELEVANT,
BANNED,
REMOVED,
};
##############################################################################################################
paginator.h
#pragma once
#include <vector>
#include <iterator>
#include <iostream>
template<typename Iterator>
class IteratorRange
{
public:
IteratorRange(Iterator begin, Iterator end, size_t size)
{
begin_ = begin;
end_ = end;
size_ = size;
}
Iterator begin() const
{
return begin_;
}
Iterator end() const
{
return end_;
}
private:
Iterator begin_;
Iterator end_;
size_t size_;
};
template<typename Iterator>
std::ostream& operator<<(std::ostream& out, const IteratorRange <Iterator>& ir)
{
for (auto it = ir.begin(); it != ir.end(); ++it)
{
out << *it;
}
return out;
}
template<typename Iterator>
IteratorRange<Iterator>& operator*(IteratorRange<Iterator> ir)
{
return ir;
}
template <typename Iterator>
class Paginator
{
public:
Paginator(Iterator range_begin, Iterator range_end, size_t sizz)
{
/*int max_size = range_end - range_begin;*/
int max_size = distance(range_begin, range_end);
int size = sizz;
for (auto it = range_begin; it != range_end;)
{
int add = min(max_size, size);
auto first = it;
/*auto second = it + add;*/
auto second = it;
advance(second, add);
ir_.push_back({ first , second, sizz });
/*it += add;*/
advance(it, add);
max_size -= size;
}
}
auto begin() const
{
return ir_.begin();
}
auto end() const
{
return ir_.end();
}
private:
vector<IteratorRange<Iterator>> ir_;
};
##############################################################################################################
read_input_functions.h
#pragma once
#include <iostream>
#include "document.h"
using namespace std;
ostream& operator<< (ostream& out, const Document& document);
##############################################################################################################
read_input_functions.cpp
#include "read_input_functions.h"
ostream& operator<< (ostream& out, const Document& document)
{
out << "{ document_id = "s << document.id << ", relevance = "s << document.relevance
<< ", rating = " << document.rating << " }"s;
return out;
}
##############################################################################################################
request_queue.h
#pragma once
#include <deque>
#include <vector>
#include "search_server.h"
using namespace std;
class RequestQueue {
public:
explicit RequestQueue(const SearchServer& search_server);
// сделаем "обёртки" для всех методов поиска, чтобы сохранять результаты для нашей статистики
template <typename DocumentPredicate>
vector<Document> AddFindRequest(const string& raw_query, DocumentPredicate document_predicate) {
vector<Document> docs = sr_.FindTopDocuments(raw_query, document_predicate);
Adjust(requests_);
QueryResult qr;
if (docs.empty())
{
qr.time_point += 1;
qr.is_empty_request = true;
}
else
{
qr.time_point += 1;
qr.is_empty_request = false;
}
requests_.push_back(qr);
return docs;
}
vector<Document> AddFindRequest(const string& raw_query, DocumentStatus status) {
vector<Document> docs = sr_.FindTopDocuments(raw_query, status);
Adjust(requests_);
QueryResult qr;
if (docs.empty())
{
qr.time_point += 1;
qr.is_empty_request = true;
}
else
{
qr.time_point += 1;
qr.is_empty_request = false;
}
requests_.push_back(qr);
return docs;
}
vector<Document> AddFindRequest(const string& raw_query) {
vector<Document> docs = sr_.FindTopDocuments(raw_query);
Adjust(requests_);
QueryResult qr;
if (docs.empty())
{
qr.time_point += 1;
qr.is_empty_request = true;
}
else
{
qr.time_point += 1;
qr.is_empty_request = false;
}
requests_.push_back(qr);
return docs;
}
int GetNoResultRequests() const;
private:
struct QueryResult
{
bool is_empty_request;
static int time_point;
};
deque<QueryResult> requests_;
const static int min_in_day_ = 1440;
const SearchServer& sr_;
void Adjust(deque<QueryResult>& requests_);
};
##############################################################################################################
request_queue.cpp
#include "request_queue.h"
using namespace std;
RequestQueue::RequestQueue(const SearchServer& search_server) :sr_(search_server) {}
// сделаем "обёртки" для всех методов поиска, чтобы сохранять результаты для нашей статистики
int RequestQueue::GetNoResultRequests() const {
return count_if(requests_.begin(), requests_.end(), [](const RequestQueue::QueryResult& qr)
{ return qr.is_empty_request == true; });
}
void RequestQueue::Adjust(deque<RequestQueue::QueryResult>& requests_)
{
if (!requests_.empty() && requests_.back().time_point >= min_in_day_)
{
requests_.pop_front();
}
}
int RequestQueue::QueryResult::time_point = 0;
##############################################################################################################
search_server.h
#pragma once
#include <iostream>
#include <algorithm>
#include <vector>
#include <map>
#include <set>
#include <cmath>
#include "string_processing.h"
#include "document.h"
using namespace std;
const int MAX_RESULT_DOCUMENT_COUNT = 5;
template <typename StringContainer>
set<string> MakeUniqueNonEmptyStrings(const StringContainer& strings) {
set<string> non_empty_strings;
for (const string& str : strings) {
if (!str.empty()) {
non_empty_strings.insert(str);
}
}
return non_empty_strings;
}
class SearchServer {
public:
template <typename StringContainer>
explicit SearchServer(const StringContainer& stop_words)
: stop_words_(MakeUniqueNonEmptyStrings(stop_words)) // Extract non-empty stop words
{
if (!all_of(stop_words_.begin(), stop_words_.end(), IsValidWord)) {
throw invalid_argument("Some of stop words are invalid"s);
}
}
explicit SearchServer(const string& stop_words_text)
: SearchServer(
SplitIntoWords(stop_words_text)) // Invoke delegating constructor from string container
{
}
void AddDocument(int document_id, const string& document, DocumentStatus status,
const vector<int>& ratings);
template <typename DocumentPredicate>
vector<Document> FindTopDocuments(const string& raw_query,
DocumentPredicate document_predicate) const {
const auto query = ParseQuery(raw_query);
auto matched_documents = FindAllDocuments(query, document_predicate);
sort(matched_documents.begin(), matched_documents.end(),
[](const Document& lhs, const Document& rhs) {
if (abs(lhs.relevance - rhs.relevance) < 1e-6) {
return lhs.rating > rhs.rating;
}
else {
return lhs.relevance > rhs.relevance;
}
});
if (matched_documents.size() > MAX_RESULT_DOCUMENT_COUNT) {
matched_documents.resize(MAX_RESULT_DOCUMENT_COUNT);
}
return matched_documents;
}
vector<Document> FindTopDocuments(const string& raw_query, DocumentStatus status) const {
return FindTopDocuments(
raw_query, [status](int document_id, DocumentStatus document_status, int rating) {
return document_status == status;
});
}
vector<Document> FindTopDocuments(const string& raw_query) const {
return FindTopDocuments(raw_query, DocumentStatus::ACTUAL);
}
int GetDocumentCount() const;
/* int GetDocumentId(int index) const;*/
tuple<vector<string>, DocumentStatus> MatchDocument(const string& raw_query,
int document_id) const;
vector<int>::const_iterator begin() const;
vector<int>::const_iterator end() const;
const map<string, double>& GetWordFrequencies(int document_id) const;
void RemoveDocument(int document_id);
map<int, pair<set<string>, bool>> MyFunc() const;
private:
struct DocumentData {
int rating;
DocumentStatus status;
};
const set<string> stop_words_;
map<string, map<int, double>> word_to_document_freqs_;
///////////////////////////////////
map<int, map<string, double>> id_to_document_freqs_;
///////////////////////////////////
map<int, pair<set<string>, bool>> id_docs;// new !!!
///////////////////////////////////
map<int, DocumentData> documents_;
vector<int> document_ids_;
bool IsStopWord(const string& word) const;
static bool IsValidWord(const string& word);
vector<string> SplitIntoWordsNoStop(const string& text) const;
static int ComputeAverageRating(const vector<int>& ratings);
struct QueryWord {
string data;
bool is_minus;
bool is_stop;
};
QueryWord ParseQueryWord(const string& text) const;
struct Query {
set<string> plus_words;
set<string> minus_words;
};
Query ParseQuery(const string& text) const;
// Existence required
double ComputeWordInverseDocumentFreq(const string& word) const;
template <typename DocumentPredicate>
vector<Document> FindAllDocuments(const Query& query,
DocumentPredicate document_predicate) const {
map<int, double> document_to_relevance;
for (const string& word : query.plus_words) {
if (word_to_document_freqs_.count(word) == 0) {
continue;
}
const double inverse_document_freq = ComputeWordInverseDocumentFreq(word);
for (const auto [document_id, term_freq] : word_to_document_freqs_.at(word)) {
const auto& document_data = documents_.at(document_id);
if (document_predicate(document_id, document_data.status, document_data.rating)) {
document_to_relevance[document_id] += term_freq * inverse_document_freq;
}
}
}
for (const string& word : query.minus_words) {
if (word_to_document_freqs_.count(word) == 0) {
continue;
}
for (const auto [document_id, _] : word_to_document_freqs_.at(word)) {
document_to_relevance.erase(document_id);
}
}
vector<Document> matched_documents;
for (const auto [document_id, relevance] : document_to_relevance) {
matched_documents.push_back(
{ document_id, relevance, documents_.at(document_id).rating });
}
return matched_documents;
}
};
##############################################################################################################
search_server.cpp
#include "search_server.h"
void SearchServer::AddDocument(int document_id, const string& document, DocumentStatus status,
const vector<int>& ratings)
{
if ((document_id < 0) || (documents_.count(document_id) > 0)) {
throw invalid_argument("Invalid document_id"s);
}
const auto words = SplitIntoWordsNoStop(document);
const double inv_word_count = 1.0 / words.size();
for (const string& word : words) {
word_to_document_freqs_[word][document_id] += inv_word_count;
}
/////////////////new
for (const string& word : words) {
id_to_document_freqs_[document_id][word] += inv_word_count;
}
////////////////new
documents_.emplace(document_id, DocumentData{ ComputeAverageRating(ratings), status });
document_ids_.push_back(document_id);
////////////////new
//map<int, pair<set<string>, bool>> id_docs;// new !!!
pair<set<string>, bool> temp_pair = { set(words.begin(), words.end()), false };
if (id_docs.empty())
{
id_docs[document_id] = temp_pair;
}
else
{
for (const auto& [id, set_bool] : id_docs)
{
if ((set_bool.first) == (temp_pair.first))
{
temp_pair.second = true;
id_docs[document_id] = temp_pair;
break;
}
}
id_docs[document_id] = temp_pair;
}
////////////////new
}
int SearchServer::GetDocumentCount() const {
return documents_.size();
}
//int SearchServer::GetDocumentId(int index) const {
// return document_ids_.at(index);
//}
tuple<vector<string>, DocumentStatus> SearchServer::MatchDocument(const string& raw_query,
int document_id) const {
const auto query = ParseQuery(raw_query);
vector<string> matched_words;
for (const string& word : query.plus_words) {
if (word_to_document_freqs_.count(word) == 0) {
continue;
}
if (word_to_document_freqs_.at(word).count(document_id)) {
matched_words.push_back(word);
}
}
for (const string& word : query.minus_words) {
if (word_to_document_freqs_.count(word) == 0) {
continue;
}
if (word_to_document_freqs_.at(word).count(document_id)) {
matched_words.clear();
break;
}
}
return { matched_words, documents_.at(document_id).status };
}
bool SearchServer::IsStopWord(const string& word) const {
return stop_words_.count(word) > 0;
}
bool SearchServer::IsValidWord(const string& word) {
// A valid word must not contain special characters
return none_of(word.begin(), word.end(), [](char c) {
return c >= '\0' && c < ' ';
});
}
vector<string> SearchServer::SplitIntoWordsNoStop(const string& text) const {
vector<string> words;
for (const string& word : SplitIntoWords(text)) {
if (!IsValidWord(word)) {
throw invalid_argument("Word "s + word + " is invalid"s);
}
if (!IsStopWord(word)) {
words.push_back(word);
}
}
return words;
}
int SearchServer::ComputeAverageRating(const vector<int>& ratings) {
if (ratings.empty()) {
return 0;
}
int rating_sum = 0;
for (const int rating : ratings) {
rating_sum += rating;
}
return rating_sum / static_cast<int>(ratings.size());
}
SearchServer::QueryWord SearchServer::ParseQueryWord(const string& text) const {
if (text.empty()) {
throw invalid_argument("Query word is empty"s);
}
string word = text;
bool is_minus = false;
if (word[0] == '-') {
is_minus = true;
word = word.substr(1);
}
if (word.empty() || word[0] == '-' || !IsValidWord(word)) {
throw invalid_argument("Query word "s + text + " is invalid");
}
return { word, is_minus, IsStopWord(word) };
}
SearchServer::Query SearchServer::ParseQuery(const string& text) const {
Query result;
for (const string& word : SplitIntoWords(text)) {
const auto query_word = ParseQueryWord(word);
if (!query_word.is_stop) {
if (query_word.is_minus) {
result.minus_words.insert(query_word.data);
}
else {
result.plus_words.insert(query_word.data);
}
}
}
return result;
}
// Existence required
double SearchServer::ComputeWordInverseDocumentFreq(const string& word) const {
return log(GetDocumentCount() * 1.0 / word_to_document_freqs_.at(word).size());
}
vector<int>::const_iterator SearchServer::begin() const
{
return document_ids_.begin();
}
vector<int>::const_iterator SearchServer::end() const
{
return document_ids_.end();
}
const map<string, double>& SearchServer::GetWordFrequencies(int document_id) const
{
static const map<string, double> empty;
if (id_to_document_freqs_.count(document_id) == 0)
{
return empty;
}
return id_to_document_freqs_.at(document_id);
}
void SearchServer::RemoveDocument(int document_id)
{
vector<string> word_to_delete;
for (const auto& [word, id_TF] : word_to_document_freqs_)
{
if (id_TF.count(document_id) != 0)
{
word_to_delete.push_back(word);
}
}
for (const auto& word : word_to_delete)
{
word_to_document_freqs_[word].erase(document_id);
}
documents_.erase(document_id);
id_docs.erase(document_id);
document_ids_.erase(remove(document_ids_.begin(), document_ids_.end(), document_id));
}
map<int, pair<set<string>, bool>> SearchServer::MyFunc() const
{
return id_docs;
}
##############################################################################################################
string_processing.h
#pragma once
#include <iostream>
#include <vector>
#include <string>
using namespace std;
string ReadLine();
int ReadLineWithNumber();
vector<string> SplitIntoWords(const string& text);
##############################################################################################################
string_processing.cpp
#include "string_processing.h"
string ReadLine() {
string s;
getline(cin, s);
return s;
}
int ReadLineWithNumber() {
int result;
cin >> result;
ReadLine();
return result;
}
vector<string> SplitIntoWords(const string& text) {
vector<string> words;
string word;
for (const char c : text) {
if (c == ' ') {
if (!word.empty()) {
words.push_back(word);
word.clear();
}
}
else {
word += c;
}
}
if (!word.empty()) {
words.push_back(word);
}
return words;
}
##############################################################################################################
remove_duplicates.h
#pragma once
#include "search_server.h"
void RemoveDuplicates(SearchServer& search_server);
##############################################################################################################
remove_duplicates.cpp
#include "remove_duplicates.h"
void RemoveDuplicates(SearchServer& search_server)
{
for (const auto& [id, set_bool] : search_server.MyFunc())
{
if (set_bool.second == true)
{
search_server.RemoveDocument(id);
cout << "Found duplicate document id "s << id << endl;
}
}
}

Add Comment

Please, Sign In to add comment