root/fileserver
1
0
Fork 0
Code Issues 11 Actions Wiki Activity

Added mutex to fs operations

Browse Source
This commit is contained in:
Mutzi 2022-09-04 05:33:26 +02:00
parent ea78585a94
commit 6159f2e940
6 changed files with 628 additions and 577 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
backend/CMakeLists.txt
View File

@ -21,9 +21,11 @@ add_executable(backend
src/controllers/controllers.h src/controllers/controllers.h
src/controllers/admin.cpp src/controllers/admin.cpp
src/controllers/fs.cpp
src/controllers/user.cpp src/controllers/user.cpp
src/controllers/fs/fs_routes.cpp
src/controllers/fs/fs_functions.cpp
src/controllers/auth/auth_common.cpp src/controllers/auth/auth_common.cpp
src/controllers/auth/auth_basic.cpp src/controllers/auth/auth_basic.cpp
src/controllers/auth/auth_2fa.cpp src/controllers/auth/auth_2fa.cpp

3
backend/src/controllers/admin.cpp
View File

@ -62,6 +62,9 @@ namespace api {
db::MapperUser user_mapper(drogon::app().getDbClient()); db::MapperUser user_mapper(drogon::app().getDbClient());
auto user = user_mapper.findByPrimaryKey(user_id); auto user = user_mapper.findByPrimaryKey(user_id);
auth::revoke_all(user); auth::revoke_all(user);
std::unique_lock lock(*fs::get_user_mutex(user.getValueOfId()));
fs::delete_node(fs::get_node(user.getValueOfRootId()).value(), chan, true); fs::delete_node(fs::get_node(user.getValueOfRootId()).value(), chan, true);
user_mapper.deleteOne(user); user_mapper.deleteOne(user);
cbk(dto::Responses::get_success_res()); cbk(dto::Responses::get_success_res());

18
backend/src/controllers/controllers.h
View File

@ -1,10 +1,14 @@
#ifndef BACKEND_CONTROLLERS_H #ifndef BACKEND_CONTROLLERS_H
#define BACKEND_CONTROLLERS_H #define BACKEND_CONTROLLERS_H
#include <variant> #include <variant>
#include <unordered_map>
#include <shared_mutex>
#include <drogon/drogon.h> #include <drogon/drogon.h>
#include <botan/rng.h> #include <botan/rng.h>
#include <msd/channel.hpp> #include <msd/channel.hpp>
#include <trantor/net/EventLoopThread.h>
#include <kubazip/zip/zip.h>
#include "db/db.h" #include "db/db.h"
@ -111,7 +115,7 @@ public:
static std::variant<db::INode, fs::create_node_error, std::tuple<bool, uint64_t>> static std::variant<db::INode, fs::create_node_error, std::tuple<bool, uint64_t>>
create_node(std::string name, const db::User& owner, bool file, const std::optional<uint64_t> &parent, bool force = false); create_node(std::string name, const db::User& owner, bool file, const std::optional<uint64_t> &parent, bool force = false);
static void delete_node(db::INode node, msd::channel<std::string>& chan, bool allow_root = false); static void delete_node(db::INode node, msd::channel<std::string>& chan, bool allow_root = false);
static std::shared_ptr<std::shared_mutex> get_user_mutex(uint64_t user_id);
void root(req_type, cbk_type); void root(req_type, cbk_type);
void node(req_type, cbk_type, uint64_t node); void node(req_type, cbk_type, uint64_t node);
@ -124,6 +128,18 @@ public:
void download_multi(req_type, cbk_type); void download_multi(req_type, cbk_type);
void download_preview(req_type, cbk_type, uint64_t node); void download_preview(req_type, cbk_type, uint64_t node);
void get_type(req_type, cbk_type, uint64_t node); void get_type(req_type, cbk_type, uint64_t node);
private:
static trantor::EventLoop* get_zip_loop();
static trantor::EventLoop* get_delete_loop();
static void generate_path(db::INode node, std::string& str);
static Json::Value generate_path(db::INode node);
static uint64_t calc_total_size(const db::INode& base);
static void add_to_zip(struct zip_t* zip, const std::string& key, const db::INode& node, const std::string& path);
static uint64_t next_temp_id;
static std::unordered_map<std::string, std::string> zip_to_temp_map;
static std::unordered_map<std::string, std::tuple<std::string, uint64_t, uint64_t>> in_progress_zips;
}; };
class user : public drogon::HttpController<user> { class user : public drogon::HttpController<user> {

257
backend/src/controllers/fs/fs_functions.cpp Normal file
View File

@ -0,0 +1,257 @@
#include <filesystem>
#include <fstream>
#include "controllers/controllers.h"
#include "dto/dto.h"
char windows_invalid_chars[] = "\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0A\x0B\x0C\x0D\x0E\x0F\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1A\x1B\x1C\x1D\x1E\x1F<>:\"/\\|";
namespace api {
uint64_t fs::next_temp_id = 0;
std::unordered_map<std::string, std::string> fs::zip_to_temp_map;
std::unordered_map<std::string, std::tuple<std::string, uint64_t, uint64_t>> fs::in_progress_zips;
std::optional<db::INode> fs::get_node(uint64_t node) {
db::MapperInode inode_mapper(drogon::app().getDbClient());
try {
return inode_mapper.findByPrimaryKey(node);
} catch (const std::exception&) {
return std::nullopt;
}
}
std::optional<db::INode> fs::get_node_and_validate(const db::User &user, uint64_t node) {
auto inode = get_node(node);
if (!inode.has_value()) return std::nullopt;
if (inode->getValueOfOwnerId() != user.getValueOfId()) return std::nullopt;
return inode;
}
std::vector<db::INode> fs::get_children(const db::INode& parent) {
db::MapperInode inode_mapper(drogon::app().getDbClient());
return inode_mapper.findBy(db::Criteria(db::INode::Cols::_parent_id, db::CompareOps::EQ, parent.getValueOfId()));
}
std::variant<db::INode, fs::create_node_error, std::tuple<bool, uint64_t>>
fs::create_node(std::string name, const db::User& owner, bool file, const std::optional<uint64_t> &parent, bool force) {
// Stolen from https://github.com/boostorg/filesystem/blob/develop/src/portability.cpp
if (!force)
if (name.empty() || name[0] == ' ' || name.find_first_of(windows_invalid_chars, 0, sizeof(windows_invalid_chars)) != std::string::npos || *(name.end() - 1) == ' ' || *(name.end() - 1) == '.' || name == "." || name == "..")
return {create_node_error::INVALID_NAME};
db::INode node;
node.setIsFile(file ? 1 : 0);
node.setName(name);
node.setOwnerId(owner.getValueOfId());
node.setHasPreview(0);
if (parent.has_value()) {
auto parent_node = get_node_and_validate(owner, *parent);
if (!parent_node.has_value())
return {create_node_error::INVALID_PARENT};
if (parent_node->getValueOfIsFile() != 0)
return {create_node_error::FILE_PARENT};
auto children = get_children(*parent_node);
for (const auto& child : children)
if (child.getValueOfName() == name)
return {std::make_tuple(
child.getValueOfIsFile() != 0,
child.getValueOfId()
)};
node.setParentId(*parent);
}
db::MapperInode inode_mapper(drogon::app().getDbClient());
inode_mapper.insert(node);
return {node};
}
void fs::delete_node(db::INode node, msd::channel<std::string>& chan, bool allow_root) {
if (node.getValueOfParentId() == 0 && (!allow_root)) return;
db::MapperInode inode_mapper(drogon::app().getDbClient());
const auto delete_file = [&chan, &inode_mapper](const db::INode& node) {
std::string entry = "Deleting ";
generate_path(node, entry);
entry >> chan;
std::filesystem::path p("./files");
p /= std::to_string(node.getValueOfId());
std::filesystem::remove(p);
if (node.getValueOfHasPreview() != 0)
std::filesystem::remove(p.string() + "_preview.png");
inode_mapper.deleteOne(node);
std::string(" Done\n") >> chan;
};
std::stack<db::INode> queue, files, folders;
if (node.getValueOfIsFile() == 0) queue.push(node);
else files.push(node);
while (!queue.empty()) {
while (!files.empty()) {
delete_file(files.top());
files.pop();
}
std::string entry = "Deleting ";
generate_path(queue.top(), entry);
entry += "\n";
entry >> chan;
auto children = get_children(queue.top());
folders.push(queue.top());
queue.pop();
for (const auto& child : children) {
if (child.getValueOfIsFile() == 0) queue.push(child);
else files.push(child);
}
}
while (!files.empty()) {
delete_file(files.top());
files.pop();
}
while (!folders.empty()) {
inode_mapper.deleteOne(folders.top());
folders.pop();
}
}
std::shared_ptr<std::shared_mutex> fs::get_user_mutex(uint64_t user_id) {
static std::unordered_map<uint64_t, std::shared_ptr<std::shared_mutex>> mutexes;
static std::mutex mutexes_mutex;
std::lock_guard guard(mutexes_mutex);
return (*mutexes.try_emplace(user_id, std::make_shared<std::shared_mutex>()).first).second;
}
trantor::EventLoop* fs::get_zip_loop() {
static bool init_done = false;
static trantor::EventLoopThread loop("ZipEventLoop");
if (!init_done) {
init_done = true;
loop.run();
loop.getLoop()->runEvery(30*60, []{
for (const auto& entry : std::filesystem::directory_iterator("./temp")) {
if (!entry.is_regular_file()) continue;
const std::string file_name = "./temp/" + entry.path().filename().string();
const auto& progress_pos = std::find_if(in_progress_zips.begin(), in_progress_zips.end(),
[&file_name](const std::pair<std::string, std::tuple<std::string, uint64_t, uint64_t>>& entry) {
return std::get<0>(entry.second) == file_name;
}
);
if (progress_pos != in_progress_zips.end()) return;
const auto& zip_map_pos = std::find_if(zip_to_temp_map.begin(), zip_to_temp_map.end(),
[&file_name](const std::pair<std::string, std::string>& entry){
return entry.second == file_name;
}
);
if (zip_map_pos != zip_to_temp_map.end()) return;
std::filesystem::remove(entry.path());
}
});
}
return loop.getLoop();
}
trantor::EventLoop* fs::get_delete_loop() {
static bool init_done = false;
static trantor::EventLoopThread loop("DeleteEventLoop");
if (!init_done) {
init_done = true;
loop.run();
}
return loop.getLoop();
}
void fs::generate_path(db::INode node, std::string& str) {
db::MapperInode inode_mapper(drogon::app().getDbClient());
std::stack<db::INode> path;
path.push(node);
while (node.getParentId() != nullptr) {
node = inode_mapper.findByPrimaryKey(node.getValueOfParentId());
path.push(node);
}
while (!path.empty()) {
const db::INode& seg = path.top();
str += seg.getValueOfName();
if (seg.getValueOfIsFile() == 0) str += "/";
path.pop();
}
}
Json::Value fs::generate_path(db::INode node) {
Json::Value segments = Json::Value(Json::ValueType::arrayValue);
db::MapperInode inode_mapper(drogon::app().getDbClient());
std::stack<db::INode> path;
path.push(node);
while (node.getParentId() != nullptr) {
node = inode_mapper.findByPrimaryKey(node.getValueOfParentId());
path.push(node);
}
while (!path.empty()) {
const db::INode& seg = path.top();
if (seg.getParentId() == nullptr) {
Json::Value json_seg;
json_seg["path"] = "/";
json_seg["node"] = seg.getValueOfId();
segments.append(json_seg);
} else {
Json::Value json_seg;
json_seg["path"] = seg.getValueOfName();
json_seg["node"] = seg.getValueOfId();
segments.append(json_seg);
if (seg.getValueOfIsFile() == 0) {
json_seg.removeMember("node");
json_seg["path"] = "/";
segments.append(json_seg);
}
}
path.pop();
}
Json::Value resp;
resp["segments"] = segments;
return resp;
}
uint64_t fs::calc_total_size(const db::INode& base) {
uint64_t size = 0;
std::stack<db::INode> queue;
queue.push(base);
while (!queue.empty()) {
const db::INode& node = queue.top();
if (node.getValueOfIsFile() == 0) {
auto children = api::fs::get_children(node);
queue.pop();
for (const auto& child : children) {
if (child.getValueOfIsFile() == 0) queue.push(child);
else if (child.getSize()) size += child.getValueOfSize();
}
} else {
size += node.getValueOfSize();
queue.pop();
}
}
return size;
}
void fs::add_to_zip(struct zip_t* zip, const std::string& key, const db::INode& node, const std::string& path) {
if (node.getValueOfIsFile() == 0) {
std::string new_path = path + node.getValueOfName() + "/";
zip_entry_opencasesensitive(zip, new_path.c_str());
zip_entry_close(zip);
auto children = api::fs::get_children(node);
for (const auto& child : children)
add_to_zip(zip, key, child, new_path);
} else {
zip_entry_opencasesensitive(zip, (path + node.getValueOfName()).c_str());
std::ifstream file("./files/" + std::to_string(node.getValueOfId()), std::ifstream::binary);
std::vector<char> buffer(64*1024);
while (!file.eof()) {
file.read(buffer.data(), (std::streamsize)buffer.size());
auto read = file.gcount();
zip_entry_write(zip, buffer.data(), read);
std::get<1>(in_progress_zips[key]) += read;
}
zip_entry_close(zip);
}
}
}

920
backend/src/controllers/fs.cpp → backend/src/controllers/fs/fs_routes.cpp
View File

@ -1,575 +1,345 @@
#pragma clang diagnostic push #include <filesystem>
#pragma ide diagnostic ignored "performance-unnecessary-value-param" #include <fstream>
#pragma ide diagnostic ignored "readability-convert-member-functions-to-static"
#include <opencv2/opencv.hpp>
#include <filesystem> #include <botan/base64.h>
#include <unordered_map>
#include <fstream> #include "controllers/controllers.h"
#include "dto/dto.h"
#include <opencv2/opencv.hpp>
#include <botan/base64.h> // https://developer.mozilla.org/en-US/docs/Web/Media/Formats/Image_types#common_image_file_types
#include <trantor/net/EventLoopThread.h> const std::unordered_map<std::string, std::string> mime_type_map = {
#include <zip/zip.h> { ".apng" , "image/apng" },
{ ".avif" , "image/avif" },
#include "controllers.h" { ".bmp" , "image/bmp" },
#include "dto/dto.h" { ".gif" , "image/gif" },
{ ".jpg" , "image/jpeg" },
char windows_invalid_chars[] = "\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0A\x0B\x0C\x0D\x0E\x0F\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1A\x1B\x1C\x1D\x1E\x1F<>:\"/\\|"; { ".jpeg" , "image/jpeg" },
{ ".jfif" , "image/jpeg" },
// https://developer.mozilla.org/en-US/docs/Web/Media/Formats/Image_types#common_image_file_types { ".pjpeg", "image/jpeg" },
const std::unordered_map<std::string, std::string> mime_type_map = { { ".pjp" , "image/jpeg" },
{ ".apng" , "image/apng" }, { ".png" , "image/png" },
{ ".avif" , "image/avif" }, { ".svg" , "image/svg" },
{ ".bmp" , "image/bmp" }, { ".webp" , "image/webp" },
{ ".gif" , "image/gif" },
{ ".jpg" , "image/jpeg" }, { ".aac" , "audio/aac" },
{ ".jpeg" , "image/jpeg" }, { ".flac" , "audio/flac" },
{ ".jfif" , "image/jpeg" }, { ".mp3" , "audio/mp3" },
{ ".pjpeg", "image/jpeg" }, { ".m4a" , "audio/mp4" },
{ ".pjp" , "image/jpeg" }, { ".oga" , "audio/ogg" },
{ ".png" , "image/png" }, { ".ogg" , "audio/ogg" },
{ ".svg" , "image/svg" }, { ".wav" , "audio/wav" },
{ ".webp" , "image/webp" },
{ ".3gp" , "video/3gpp" },
{ ".aac" , "audio/aac" }, { ".mpg" , "video/mpeg" },
{ ".flac" , "audio/flac" }, { ".mpeg" , "video/mpeg" },
{ ".mp3" , "audio/mp3" }, { ".mp4" , "video/mp4" },
{ ".m4a" , "audio/mp4" }, { ".m4v" , "video/mp4" },
{ ".oga" , "audio/ogg" }, { ".m4p" , "video/mp4" },
{ ".ogg" , "audio/ogg" }, { ".ogv" , "video/ogg" },
{ ".wav" , "audio/wav" }, { ".mov" , "video/quicktime" },
{ ".webm" , "video/webm" },
{ ".3gp" , "video/3gpp" }, { ".mkv" , "video/x-matroska" },
{ ".mpg" , "video/mpeg" }, { ".mk3d" , "video/x-matroska" },
{ ".mpeg" , "video/mpeg" }, { ".mks" , "video/x-matroska" },
{ ".mp4" , "video/mp4" }, };
{ ".m4v" , "video/mp4" },
{ ".m4p" , "video/mp4" }, template<typename InputIt>
{ ".ogv" , "video/ogg" }, std::string join_string(InputIt first, InputIt last, const std::string& separator = ",") {
{ ".mov" , "video/quicktime" }, std::ostringstream result;
{ ".webm" , "video/webm" }, if (first != last) {
{ ".mkv" , "video/x-matroska" }, result << *first;
{ ".mk3d" , "video/x-matroska" }, while (++first != last) {
{ ".mks" , "video/x-matroska" }, result << separator << *first;
}; }
}
uint64_t next_temp_id = 0; return result.str();
std::unordered_map<std::string, std::string> zip_to_temp_map; }
std::unordered_map<std::string, std::tuple<std::string, uint64_t, uint64_t>> in_progress_zips;
namespace api {
trantor::EventLoop* get_zip_loop() { void fs::root(req_type req, cbk_type cbk) {
static bool init_done = false; db::User user = dto::get_user(req);
static trantor::EventLoopThread loop("ZipEventLoop"); cbk(dto::Responses::get_root_res(user.getValueOfRootId()));
if (!init_done) { }
init_done = true;
loop.run(); void fs::node(req_type req, cbk_type cbk, uint64_t node) {
loop.getLoop()->runEvery(30*60, []{ db::User user = dto::get_user(req);
for (const auto& entry : std::filesystem::directory_iterator("./temp")) { std::shared_lock lock(*get_user_mutex(user.getValueOfId()));
if (!entry.is_regular_file()) continue; auto inode = get_node_and_validate(user, node);
const std::string file_name = "./temp/" + entry.path().filename().string(); if (!inode.has_value())
const auto& progress_pos = std::find_if(in_progress_zips.begin(), in_progress_zips.end(), return cbk(dto::Responses::get_badreq_res("Unknown node"));
[&file_name](const std::pair<std::string, std::tuple<std::string, uint64_t, uint64_t>>& entry) { auto dto_node = dto::Responses::GetNodeEntry(*inode);
return std::get<0>(entry.second) == file_name; std::vector<dto::Responses::GetNodeEntry> children;
} if (!dto_node.is_file) for (const db::INode& child : get_children(*inode)) children.emplace_back(child);
); cbk(dto::Responses::get_node_res(dto_node, children));
if (progress_pos != in_progress_zips.end()) return; }
const auto& zip_map_pos = std::find_if(zip_to_temp_map.begin(), zip_to_temp_map.end(),
[&file_name](const std::pair<std::string, std::string>& entry){ void fs::path(req_type req, cbk_type cbk, uint64_t node) {
return entry.second == file_name; db::User user = dto::get_user(req);
} std::shared_lock lock(*get_user_mutex(user.getValueOfId()));
); auto inode = get_node_and_validate(user, node);
if (zip_map_pos != zip_to_temp_map.end()) return; if (!inode.has_value())
std::filesystem::remove(entry.path()); cbk(dto::Responses::get_badreq_res("Unknown node"));
} else {
}); auto path = generate_path(*inode);
} cbk(dto::Responses::get_success_res(path));
return loop.getLoop(); }
} }
trantor::EventLoop* get_delete_loop() { template<bool file>
static bool init_done = false; void fs::create_node_req(req_type req, cbk_type cbk) {
static trantor::EventLoopThread loop("DeleteEventLoop"); db::User user = dto::get_user(req);
if (!init_done) { Json::Value& json = *req->jsonObject();
init_done = true; try {
loop.run(); uint64_t parent = dto::json_get<uint64_t>(json, "parent").value();
} std::string name = dto::json_get<std::string>(json, "name").value();
return loop.getLoop();
} std::shared_lock lock(*get_user_mutex(user.getValueOfId()));
void generate_path(db::INode node, std::string& str) { auto new_node = create_node(name, user, file, std::make_optional(parent));
db::MapperInode inode_mapper(drogon::app().getDbClient()); if (std::holds_alternative<db::INode>(new_node))
std::stack<db::INode> path; cbk(dto::Responses::get_new_node_res(std::get<db::INode>(new_node).getValueOfId()));
path.push(node); else if (std::holds_alternative<create_node_error>(new_node))
while (node.getParentId() != nullptr) { switch (std::get<create_node_error>(new_node)) {
node = inode_mapper.findByPrimaryKey(node.getValueOfParentId()); case create_node_error::INVALID_NAME: return cbk(dto::Responses::get_badreq_res("Invalid name"));
path.push(node); case create_node_error::INVALID_PARENT: return cbk(dto::Responses::get_badreq_res("Invalid parent"));
} case create_node_error::FILE_PARENT: return cbk(dto::Responses::get_badreq_res("Parent is file"));
while (!path.empty()) { }
const db::INode& seg = path.top(); else {
str += seg.getValueOfName(); auto tuple = std::get<std::tuple<bool, uint64_t>>(new_node);
if (seg.getValueOfIsFile() == 0) str += "/"; cbk(dto::Responses::get_node_exists_res(std::get<1>(tuple), std::get<0>(tuple)));
path.pop(); }
} } catch (const std::exception&) {
} cbk(dto::Responses::get_badreq_res("Validation error"));
}
Json::Value generate_path(db::INode node) { }
Json::Value segments = Json::Value(Json::ValueType::arrayValue);
db::MapperInode inode_mapper(drogon::app().getDbClient()); void fs::delete_node_req(req_type req, cbk_type cbk, uint64_t node) {
std::stack<db::INode> path; db::User user = dto::get_user(req);
path.push(node); std::unique_lock lock(*get_user_mutex(user.getValueOfId()));
while (node.getParentId() != nullptr) { auto inode = get_node_and_validate(user, node);
node = inode_mapper.findByPrimaryKey(node.getValueOfParentId()); if (!inode.has_value())
path.push(node); cbk(dto::Responses::get_badreq_res("Unknown node"));
} else if (inode->getValueOfParentId() == 0)
while (!path.empty()) { cbk(dto::Responses::get_badreq_res("Can't delete root"));
const db::INode& seg = path.top(); else {
if (seg.getParentId() == nullptr) { auto chan = std::make_shared<msd::channel<std::string>>();
Json::Value json_seg; std::string("Waiting in queue...\n") >> (*chan);
json_seg["path"] = "/"; get_delete_loop()->queueInLoop([chan, inode=*inode, user=user.getValueOfId()]{
json_seg["node"] = seg.getValueOfId(); std::unique_lock lock(*get_user_mutex(user));
segments.append(json_seg); delete_node(inode, *chan);
} else { chan->close();
Json::Value json_seg; });
json_seg["path"] = seg.getValueOfName(); cbk(drogon::HttpResponse::newStreamResponse([chan](char* buf, std::size_t size) -> std::size_t{
json_seg["node"] = seg.getValueOfId(); if (buf == nullptr) return 0;
segments.append(json_seg); if (chan->closed() && chan->empty()) return 0;
if (seg.getValueOfIsFile() == 0) { std::string buffer;
json_seg.removeMember("node"); buffer << *chan;
json_seg["path"] = "/"; if (buffer.empty()) return 0;
segments.append(json_seg); std::size_t read = std::min(size, buffer.size());
} std::memcpy(buf, buffer.data(), read); // NOLINT(bugprone-not-null-terminated-result)
} return read;
path.pop(); }));
} }
Json::Value resp; }
resp["segments"] = segments;
return resp; void fs::upload(req_type req, cbk_type cbk, uint64_t node) {
} constexpr int image_height = 256;
db::User user = dto::get_user(req);
uint64_t calc_total_size(const db::INode& base) {
uint64_t size = 0; std::shared_lock lock(*get_user_mutex(user.getValueOfId()));
std::stack<db::INode> queue;
queue.push(base); auto inode = get_node_and_validate(user, node);
while (!queue.empty()) { if (!inode.has_value())
const db::INode& node = queue.top(); return cbk(dto::Responses::get_badreq_res("Unknown node"));
if (node.getValueOfIsFile() == 0) { if (inode->getValueOfIsFile() == 0)
auto children = api::fs::get_children(node); return cbk(dto::Responses::get_badreq_res("Can't upload to a directory"));
queue.pop();
for (const auto& child : children) { drogon::MultiPartParser mpp;
if (child.getValueOfIsFile() == 0) queue.push(child); if (mpp.parse(req) != 0)
else if (child.getSize()) size += child.getValueOfSize(); return cbk(dto::Responses::get_badreq_res("Failed to parse files"));
} if (mpp.getFiles().size() != 1)
} else { return cbk(dto::Responses::get_badreq_res("Exactly 1 file needed"));
size += node.getValueOfSize();
queue.pop(); const drogon::HttpFile& file = mpp.getFiles().at(0);
}
} std::filesystem::path p("./files");
return size; p /= std::to_string(inode->getValueOfId());
}
file.saveAs(p.string());
void add_to_zip(struct zip_t* zip, const std::string& key, const db::INode& node, const std::string& path) { try {
if (node.getValueOfIsFile() == 0) { if (file.fileLength() > 100 * 1024 * 1024) throw std::exception();
std::string new_path = path + node.getValueOfName() + "/"; std::filesystem::path filename(inode->getValueOfName());
zip_entry_opencasesensitive(zip, new_path.c_str()); const std::string& mime = mime_type_map.at(filename.extension().string());
zip_entry_close(zip); if (!mime.starts_with("image")) throw std::exception();
auto children = api::fs::get_children(node);
for (const auto& child : children) cv::_InputArray image_arr(file.fileData(), (int) file.fileLength());
add_to_zip(zip, key, child, new_path); cv::Mat image = cv::imdecode(image_arr, cv::IMREAD_COLOR);
} else { if (!image.empty()) {
zip_entry_opencasesensitive(zip, (path + node.getValueOfName()).c_str()); float h_ration = ((float) image_height) / ((float) image.rows);
std::ifstream file("./files/" + std::to_string(node.getValueOfId()), std::ifstream::binary); cv::Mat preview;
std::vector<char> buffer(64*1024); cv::resize(image, preview, cv::Size((int) (((float) image.cols) * h_ration), image_height), 0, 0, cv::INTER_AREA);
while (!file.eof()) { cv::imwrite(p.string() + "_preview.png", preview);
file.read(buffer.data(), (std::streamsize)buffer.size()); inode->setHasPreview(1);
auto read = file.gcount(); }
zip_entry_write(zip, buffer.data(), read); } catch (const std::exception&) {}
std::get<1>(in_progress_zips[key]) += read; inode->setSize(file.fileLength());
}
zip_entry_close(zip); db::MapperInode inode_mapper(drogon::app().getDbClient());
} inode_mapper.update(*inode);
}
cbk(dto::Responses::get_success_res());
template<typename InputIt> }
std::string join_string(InputIt first, InputIt last, const std::string& separator = ",") {
std::ostringstream result; void fs::create_zip(req_type req, cbk_type cbk) {
if (first != last) { db::User user = dto::get_user(req);
result << *first;
while (++first != last) { std::shared_lock lock(*get_user_mutex(user.getValueOfId()));
result << separator << *first;
} Json::Value& json = *req->jsonObject();
} try {
return result.str(); if (!json.isMember("nodes")) throw std::exception();
} Json::Value node_arr = json["nodes"];
if (!node_arr.isArray()) throw std::exception();
namespace api { std::vector<uint64_t> node_ids;
std::optional<db::INode> fs::get_node(uint64_t node) { for (const auto& node : node_arr)
db::MapperInode inode_mapper(drogon::app().getDbClient()); node_ids.push_back(node.asUInt64());
try {
return inode_mapper.findByPrimaryKey(node); std::vector<db::INode> nodes;
} catch (const std::exception&) { std::transform(node_ids.begin(), node_ids.end(), std::back_inserter(nodes), [&user](uint64_t node) {
return std::nullopt; return api::fs::get_node_and_validate(user, node).value();
} });
}
std::string key = join_string(node_ids.begin(), node_ids.end());
std::optional<db::INode> fs::get_node_and_validate(const db::User &user, uint64_t node) {
auto inode = get_node(node); if (zip_to_temp_map.contains(key)) return cbk(dto::Responses::get_create_zip_done_res());
if (!inode.has_value()) return std::nullopt; if (in_progress_zips.contains(key)) {
if (inode->getValueOfOwnerId() != user.getValueOfId()) return std::nullopt; auto progress = in_progress_zips.at(key);
return inode; return cbk(dto::Responses::get_create_zip_done_res(std::get<1>(progress), std::get<2>(progress)));
} }
std::string file_name = "./temp/fs_" + std::to_string(next_temp_id++) + ".zip";
std::vector<db::INode> fs::get_children(const db::INode& parent) { in_progress_zips.emplace(key, std::make_tuple(file_name, 0, 1));
db::MapperInode inode_mapper(drogon::app().getDbClient()); get_zip_loop()->queueInLoop([key = std::move(key), nodes = std::move(nodes), file_name = std::move(file_name), user=user.getValueOfId()]{
return inode_mapper.findBy(db::Criteria(db::INode::Cols::_parent_id, db::CompareOps::EQ, parent.getValueOfId())); {
} std::shared_lock lock(*get_user_mutex(user));
uint64_t size = 0;
std::variant<db::INode, fs::create_node_error, std::tuple<bool, uint64_t>> for (const auto& node : nodes) size += calc_total_size(node);
fs::create_node(std::string name, const db::User& owner, bool file, const std::optional<uint64_t> &parent, bool force) { std::get<2>(in_progress_zips.at(key)) = size;
// Stolen from https://github.com/boostorg/filesystem/blob/develop/src/portability.cpp struct zip_t* zip = zip_open(file_name.c_str(), ZIP_DEFAULT_COMPRESSION_LEVEL, 'w');
if (!force) for (const db::INode& node : nodes)
if (name.empty() || name[0] == ' ' || name.find_first_of(windows_invalid_chars, 0, sizeof(windows_invalid_chars)) != std::string::npos || *(name.end() - 1) == ' ' || *(name.end() - 1) == '.' || name == "." || name == "..") add_to_zip(zip, key, node, "");
return {create_node_error::INVALID_NAME}; zip_close(zip);
}
db::INode node; zip_to_temp_map.emplace(key, file_name);
node.setIsFile(file ? 1 : 0); in_progress_zips.erase(key);
node.setName(name); });
node.setOwnerId(owner.getValueOfId()); return cbk(dto::Responses::get_create_zip_done_res(0, 1));
node.setHasPreview(0); } catch (const std::exception&) {
if (parent.has_value()) { cbk(dto::Responses::get_badreq_res("Validation error"));
auto parent_node = get_node_and_validate(owner, *parent); }
if (!parent_node.has_value()) }
return {create_node_error::INVALID_PARENT};
if (parent_node->getValueOfIsFile() != 0) void fs::download(req_type req, cbk_type cbk) {
return {create_node_error::FILE_PARENT}; db::User user = dto::get_user(req);
auto children = get_children(*parent_node);
for (const auto& child : children) std::shared_lock lock(*get_user_mutex(user.getValueOfId()));
if (child.getValueOfName() == name)
return {std::make_tuple( auto node_id = req->getOptionalParameter<uint64_t>("id");
child.getValueOfIsFile() != 0, if (!node_id.has_value()) {
child.getValueOfId() cbk(dto::Responses::get_badreq_res("Invalid node"));
)}; return;
node.setParentId(*parent); }
} auto inode = get_node_and_validate(user, *node_id);
db::MapperInode inode_mapper(drogon::app().getDbClient()); if (!inode.has_value()) {
inode_mapper.insert(node); cbk(dto::Responses::get_badreq_res("Invalid node"));
return {node}; return;
} }
void fs::delete_node(db::INode node, msd::channel<std::string>& chan, bool allow_root) { if (inode->getValueOfIsFile() != 0) {
if (node.getValueOfParentId() == 0 && (!allow_root)) return; std::filesystem::path p("./files");
p /= std::to_string(inode->getValueOfId());
db::MapperInode inode_mapper(drogon::app().getDbClient());
cbk(drogon::HttpResponse::newFileResponse(
const auto delete_file = [&chan, &inode_mapper](const db::INode& node) { p.string(),
std::string entry = "Deleting "; inode->getValueOfName()
generate_path(node, entry); ));
entry >> chan; } else {
std::filesystem::path p("./files"); try {
p /= std::to_string(node.getValueOfId()); std::string key = std::to_string(inode->getValueOfId());
std::filesystem::remove(p); std::string file = zip_to_temp_map.at(key);
if (node.getValueOfHasPreview() != 0) zip_to_temp_map.erase(key);
std::filesystem::remove(p.string() + "_preview.png"); cbk(drogon::HttpResponse::newFileResponse(
inode_mapper.deleteOne(node); file,
std::string(" Done\n") >> chan; inode->getValueOfName() + ".zip"
}; ));
} catch (const std::exception&) {
std::stack<db::INode> queue, files, folders; cbk(dto::Responses::get_badreq_res("Invalid node"));
}
if (node.getValueOfIsFile() == 0) queue.push(node); }
else files.push(node); }
while (!queue.empty()) { void fs::download_multi(req_type req, cbk_type cbk) {
while (!files.empty()) { db::User user = dto::get_user(req);
delete_file(files.top());
files.pop(); std::shared_lock lock(*get_user_mutex(user.getValueOfId()));
}
std::string entry = "Deleting "; auto node_ids_str = req->getOptionalParameter<std::string>("id");
generate_path(queue.top(), entry); if (!node_ids_str.has_value())
entry += "\n"; return cbk(dto::Responses::get_badreq_res("No nodes"));
entry >> chan;
auto children = get_children(queue.top()); std::stringstream node_ids_ss(*node_ids_str);
folders.push(queue.top()); std::string temp;
queue.pop(); try {
for (const auto& child : children) { while (std::getline(node_ids_ss, temp, ','))
if (child.getValueOfIsFile() == 0) queue.push(child); if (!get_node_and_validate(user, std::stoull(temp)).has_value()) throw std::exception();
else files.push(child);
} std::string file = zip_to_temp_map.at(*node_ids_str);
} zip_to_temp_map.erase(*node_ids_str);
cbk(drogon::HttpResponse::newFileResponse(
while (!files.empty()) { file,
delete_file(files.top()); "files.zip"
files.pop(); ));
} } catch (const std::exception&) {
cbk(dto::Responses::get_badreq_res("Invalid nodes"));
while (!folders.empty()) { }
inode_mapper.deleteOne(folders.top()); }
folders.pop();
} void fs::download_preview(req_type req, cbk_type cbk, uint64_t node) {
} db::User user = dto::get_user(req);
void fs::root(req_type req, cbk_type cbk) { std::shared_lock lock(*get_user_mutex(user.getValueOfId()));
db::User user = dto::get_user(req);
cbk(dto::Responses::get_root_res(user.getValueOfRootId())); auto inode = get_node_and_validate(user, node);
} if (!inode.has_value())
return cbk(dto::Responses::get_badreq_res("Unknown node"));
void fs::node(req_type req, cbk_type cbk, uint64_t node) { if (inode->getValueOfHasPreview() == 0)
db::User user = dto::get_user(req); return cbk(dto::Responses::get_badreq_res("No preview"));
auto inode = get_node_and_validate(user, node);
if (!inode.has_value()) std::filesystem::path p("./files");
return cbk(dto::Responses::get_badreq_res("Unknown node")); p /= std::to_string(inode->getValueOfId()) + "_preview.png";
auto dto_node = dto::Responses::GetNodeEntry(*inode); std::ifstream file(p, std::ios::in | std::ios::binary);
std::vector<dto::Responses::GetNodeEntry> children; std::vector<uint8_t> image((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());
if (!dto_node.is_file) for (const db::INode& child : get_children(*inode)) children.emplace_back(child);
cbk(dto::Responses::get_node_res(dto_node, children)); cbk(dto::Responses::get_download_base64_res("data:image/png;base64," + Botan::base64_encode(image)));
} }
void fs::path(req_type req, cbk_type cbk, uint64_t node) { void fs::get_type(req_type req, cbk_type cbk, uint64_t node){
db::User user = dto::get_user(req); db::User user = dto::get_user(req);
auto inode = get_node_and_validate(user, node);
if (!inode.has_value()) std::shared_lock lock(*get_user_mutex(user.getValueOfId()));
cbk(dto::Responses::get_badreq_res("Unknown node"));
else { auto inode = get_node_and_validate(user, node);
auto path = generate_path(*inode); if (!inode.has_value())
cbk(dto::Responses::get_success_res(path)); return cbk(dto::Responses::get_badreq_res("Unknown node"));
}
}
std::filesystem::path p("./files"), name(inode->getValueOfName());
template<bool file> p /= std::to_string(inode->getValueOfId());
void fs::create_node_req(req_type req, cbk_type cbk) {
db::User user = dto::get_user(req); try {
Json::Value& json = *req->jsonObject(); cbk(dto::Responses::get_type_res(mime_type_map.at(name.extension().string())));
try { } catch (const std::exception&) {
uint64_t parent = dto::json_get<uint64_t>(json, "parent").value(); cbk(dto::Responses::get_badreq_res("Invalid file type"));
std::string name = dto::json_get<std::string>(json, "name").value(); }
}
auto new_node = create_node(name, user, file, std::make_optional(parent)); }
if (std::holds_alternative<db::INode>(new_node))
cbk(dto::Responses::get_new_node_res(std::get<db::INode>(new_node).getValueOfId()));
else if (std::holds_alternative<create_node_error>(new_node))
switch (std::get<create_node_error>(new_node)) {
case create_node_error::INVALID_NAME: return cbk(dto::Responses::get_badreq_res("Invalid name"));
case create_node_error::INVALID_PARENT: return cbk(dto::Responses::get_badreq_res("Invalid parent"));
case create_node_error::FILE_PARENT: return cbk(dto::Responses::get_badreq_res("Parent is file"));
}
else {
auto tuple = std::get<std::tuple<bool, uint64_t>>(new_node);
cbk(dto::Responses::get_node_exists_res(std::get<1>(tuple), std::get<0>(tuple)));
}
} catch (const std::exception&) {
cbk(dto::Responses::get_badreq_res("Validation error"));
}
}
void fs::delete_node_req(req_type req, cbk_type cbk, uint64_t node) {
db::User user = dto::get_user(req);
auto inode = get_node_and_validate(user, node);
if (!inode.has_value())
cbk(dto::Responses::get_badreq_res("Unknown node"));
else if (inode->getValueOfParentId() == 0)
cbk(dto::Responses::get_badreq_res("Can't delete root"));
else {
auto chan = std::make_shared<msd::channel<std::string>>();
std::string("Waiting in queue...\n") >> (*chan);
get_delete_loop()->queueInLoop([chan, inode=*inode]{
delete_node(inode, *chan);
chan->close();
});
cbk(drogon::HttpResponse::newStreamResponse([chan](char* buf, std::size_t size) -> std::size_t{
if (buf == nullptr) return 0;
if (chan->closed() && chan->empty()) return 0;
std::string buffer;
buffer << *chan;
if (buffer.empty()) return 0;
std::size_t read = std::min(size, buffer.size());
std::memcpy(buf, buffer.data(), read); // NOLINT(bugprone-not-null-terminated-result)
return read;
}));
}
}
void fs::upload(req_type req, cbk_type cbk, uint64_t node) {
constexpr int image_height = 256;
db::User user = dto::get_user(req);
auto inode = get_node_and_validate(user, node);
if (!inode.has_value())
return cbk(dto::Responses::get_badreq_res("Unknown node"));
if (inode->getValueOfIsFile() == 0)
return cbk(dto::Responses::get_badreq_res("Can't upload to a directory"));
drogon::MultiPartParser mpp;
if (mpp.parse(req) != 0)
return cbk(dto::Responses::get_badreq_res("Failed to parse files"));
if (mpp.getFiles().size() != 1)
return cbk(dto::Responses::get_badreq_res("Exactly 1 file needed"));
const drogon::HttpFile& file = mpp.getFiles().at(0);
std::filesystem::path p("./files");
p /= std::to_string(inode->getValueOfId());
file.saveAs(p.string());
try {
if (file.fileLength() > 100 * 1024 * 1024) throw std::exception();
std::filesystem::path filename(inode->getValueOfName());
const std::string& mime = mime_type_map.at(filename.extension().string());
if (!mime.starts_with("image")) throw std::exception();
cv::_InputArray image_arr(file.fileData(), (int) file.fileLength());
cv::Mat image = cv::imdecode(image_arr, cv::IMREAD_COLOR);
if (!image.empty()) {
float h_ration = ((float) image_height) / ((float) image.rows);
cv::Mat preview;
cv::resize(image, preview, cv::Size((int) (((float) image.cols) * h_ration), image_height), 0, 0, cv::INTER_AREA);
cv::imwrite(p.string() + "_preview.png", preview);
inode->setHasPreview(1);
}
} catch (const std::exception&) {}
inode->setSize(file.fileLength());
db::MapperInode inode_mapper(drogon::app().getDbClient());
inode_mapper.update(*inode);
cbk(dto::Responses::get_success_res());
}
void fs::create_zip(req_type req, cbk_type cbk) {
db::User user = dto::get_user(req);
Json::Value& json = *req->jsonObject();
try {
if (!json.isMember("nodes")) throw std::exception();
Json::Value node_arr = json["nodes"];
if (!node_arr.isArray()) throw std::exception();
std::vector<uint64_t> node_ids;
for (const auto& node : node_arr)
node_ids.push_back(node.asUInt64());
std::vector<db::INode> nodes;
std::transform(node_ids.begin(), node_ids.end(), std::back_inserter(nodes), [&user](uint64_t node) {
return api::fs::get_node_and_validate(user, node).value();
});
std::string key = join_string(node_ids.begin(), node_ids.end());
if (zip_to_temp_map.contains(key)) return cbk(dto::Responses::get_create_zip_done_res());
if (in_progress_zips.contains(key)) {
auto progress = in_progress_zips.at(key);
return cbk(dto::Responses::get_create_zip_done_res(std::get<1>(progress), std::get<2>(progress)));
}
uint64_t size = 0;
for (const auto& node : nodes) size += calc_total_size(node);
std::string file_name = "./temp/fs_" + std::to_string(next_temp_id++) + ".zip";
in_progress_zips.emplace(key, std::make_tuple(file_name, 0, size));
get_zip_loop()->queueInLoop([key = std::move(key), nodes = std::move(nodes), file_name = std::move(file_name)]{
{
struct zip_t* zip = zip_open(file_name.c_str(), ZIP_DEFAULT_COMPRESSION_LEVEL, 'w');
for (const db::INode& node : nodes)
add_to_zip(zip, key, node, "");
zip_close(zip);
}
zip_to_temp_map.emplace(key, file_name);
in_progress_zips.erase(key);
});
return cbk(dto::Responses::get_create_zip_done_res(0, size));
} catch (const std::exception&) {
cbk(dto::Responses::get_badreq_res("Validation error"));
}
}
void fs::download(req_type req, cbk_type cbk) {
db::User user = dto::get_user(req);
auto node_id = req->getOptionalParameter<uint64_t>("id");
if (!node_id.has_value()) {
cbk(dto::Responses::get_badreq_res("Invalid node"));
return;
}
auto inode = get_node_and_validate(user, *node_id);
if (!inode.has_value()) {
cbk(dto::Responses::get_badreq_res("Invalid node"));
return;
}
if (inode->getValueOfIsFile() != 0) {
std::filesystem::path p("./files");
p /= std::to_string(inode->getValueOfId());
cbk(drogon::HttpResponse::newFileResponse(
p.string(),
inode->getValueOfName()
));
} else {
try {
std::string key = std::to_string(inode->getValueOfId());
std::string file = zip_to_temp_map.at(key);
zip_to_temp_map.erase(key);
cbk(drogon::HttpResponse::newFileResponse(
file,
inode->getValueOfName() + ".zip"
));
} catch (const std::exception&) {
cbk(dto::Responses::get_badreq_res("Invalid node"));
}
}
}
void fs::download_multi(req_type req, cbk_type cbk) {
db::User user = dto::get_user(req);
auto node_ids_str = req->getOptionalParameter<std::string>("id");
if (!node_ids_str.has_value())
return cbk(dto::Responses::get_badreq_res("No nodes"));
std::stringstream node_ids_ss(*node_ids_str);
std::string temp;
try {
while (std::getline(node_ids_ss, temp, ','))
if (!get_node_and_validate(user, std::stoull(temp)).has_value()) throw std::exception();
std::string file = zip_to_temp_map.at(*node_ids_str);
zip_to_temp_map.erase(*node_ids_str);
cbk(drogon::HttpResponse::newFileResponse(
file,
"files.zip"
));
} catch (const std::exception&) {
cbk(dto::Responses::get_badreq_res("Invalid nodes"));
}
}
void fs::download_preview(req_type req, cbk_type cbk, uint64_t node) {
db::User user = dto::get_user(req);
auto inode = get_node_and_validate(user, node);
if (!inode.has_value())
return cbk(dto::Responses::get_badreq_res("Unknown node"));
if (inode->getValueOfHasPreview() == 0)
return cbk(dto::Responses::get_badreq_res("No preview"));
std::filesystem::path p("./files");
p /= std::to_string(inode->getValueOfId()) + "_preview.png";
std::ifstream file(p, std::ios::in | std::ios::binary);
std::vector<uint8_t> image((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());
cbk(dto::Responses::get_download_base64_res("data:image/png;base64," + Botan::base64_encode(image)));
}
void fs::get_type(req_type req, cbk_type cbk, uint64_t node){
db::User user = dto::get_user(req);
auto inode = get_node_and_validate(user, node);
if (!inode.has_value())
return cbk(dto::Responses::get_badreq_res("Unknown node"));
std::filesystem::path p("./files"), name(inode->getValueOfName());
p /= std::to_string(inode->getValueOfId());
try {
cbk(dto::Responses::get_type_res(mime_type_map.at(name.extension().string())));
} catch (const std::exception&) {
cbk(dto::Responses::get_badreq_res("Invalid file type"));
}
}
}
#pragma clang diagnostic pop

3
backend/src/controllers/user.cpp
View File

@ -21,6 +21,9 @@ namespace api {
msd::channel<std::string> chan; msd::channel<std::string> chan;
db::User user = dto::get_user(req); db::User user = dto::get_user(req);
auth::revoke_all(user); auth::revoke_all(user);
std::unique_lock lock(*fs::get_user_mutex(user.getValueOfId()));
fs::delete_node((fs::get_node(user.getValueOfRootId())).value(), chan, true); fs::delete_node((fs::get_node(user.getValueOfRootId())).value(), chan, true);
user_mapper.deleteOne(user); user_mapper.deleteOne(user);