refactor(wip): filesystem to avoid fragmentation

in many places, you had to have certain files in certain folders
autoreset / api key / ihud images / tas folder in Portal 2
crosshairs in portal2
cfgs wouldnt autocomplete from p2common
etc

now you can put anything anywhere and it just works ™️
if a feature tries to read a file it can do it from any search path
and if it writes it will first try to overwrite an existing
e.g.

Author: ThisAMJ

src/Checksum.cpp

@@ -3,6 +3,7 @@
 #include "Utils.hpp"
 #include "Event.hpp"
 #include "Modules/Engine.hpp"
+#include "Modules/FileSystem.hpp"
 #include "Utils/ed25519/ed25519.h"
 #include "Version.hpp"
 
@@ -257,20 +258,33 @@ static void calcFileSums(std::map<std::string, uint32_t> *out, std::vector<std::
 static void initFileSums() {
 	std::vector<std::string> paths;
 	try {
-		for (auto &ent : std::filesystem::recursive_directory_iterator(".")) {
-			if (ent.status().type() == std::filesystem::file_type::regular || ent.status().type() == std::filesystem::file_type::symlink) {
-				auto path = ent.path().string();
-				std::replace(path.begin(), path.end(), '\\', '/');
-
-				bool dlc = path.find("portal2_dlc") != std::string::npos &&
-					path.find("portal2_dlc1") == std::string::npos &&
-					path.find("portal2_dlc2") == std::string::npos;
-
-				if (Utils::EndsWith(path, ".nut")
-					|| (Utils::EndsWith(path, ".vpk") && dlc)
-					|| path.find("scripts/talker") != std::string::npos)
-				{
-					paths.push_back(path);
+		auto searchpaths = fileSystem->GetSearchPaths();
+		for (auto searchpath : searchpaths) {
+			auto iterator = std::filesystem::recursive_directory_iterator(searchpath, std::filesystem::directory_options::follow_directory_symlink);
+			for (auto &ent : iterator) {
+				if (ent.status().type() == std::filesystem::file_type::directory) {
+					// skip directories that are other search paths
+					for (auto otherpath : searchpaths) {
+						if (std::filesystem::equivalent(ent.path(), otherpath)) {
+							iterator.disable_recursion_pending();
+							break;
+						}
+					}
+				}
+				if (ent.status().type() == std::filesystem::file_type::regular || ent.status().type() == std::filesystem::file_type::symlink) {
+					auto path = ent.path().string();
+					std::replace(path.begin(), path.end(), '\\', '/');
+
+					bool dlc = path.find("portal2_dlc") != std::string::npos &&
+						path.find("portal2_dlc1") == std::string::npos &&
+						path.find("portal2_dlc2") == std::string::npos;
+
+					if (Utils::EndsWith(path, ".nut")
+						|| (Utils::EndsWith(path, ".vpk") && dlc)
+						|| path.find("scripts/talker") != std::string::npos)
+					{
+						paths.push_back(path);
+					}
 				}
 			}
 		}

src/Command.cpp

@@ -1,6 +1,7 @@
 #include "Command.hpp"
 
 #include "Game.hpp"
+#include "Modules/FileSystem.hpp"
 #include "Modules/Tier1.hpp"
 #include "SAR.hpp"
 
@@ -219,15 +220,32 @@ int _FileCompletionFunc(std::string extension, std::string rootdir, int exp_args
 	try {
 		std::set<std::string> sorted;
 
-		for (auto &file : std::filesystem::directory_iterator(rootdir + std::string("/") + dirpart)) {
-			try {
-				if (file.is_directory() || Utils::EndsWith(file.path().extension().string(), extension)) {
-					std::string path = dirpart + file.path().stem().string();
-					std::replace(path.begin(), path.end(), '\\', '/');
-					if (file.is_directory()) path += "/";
-					sorted.insert(path);
+		auto gamedirs = fileSystem->GetSearchPaths();
+		for (auto gamedir : gamedirs) {
+			if (std::filesystem::is_directory(gamedir + rootdir + "/" + dirpart)) {
+				for (auto &file : std::filesystem::directory_iterator(gamedir + rootdir + "/" + dirpart)) {
+					try {
+						if (file.is_directory() || Utils::EndsWith(file.path().extension().string(), extension)) {
+							std::string path = dirpart + file.path().stem().string();
+							std::replace(path.begin(), path.end(), '\\', '/');
+							if (file.is_directory()) {
+								path += "/";
+								// This is a bit of a hack, but it works
+								// avoids confusion such as "do_stuff portal2/..." == "do_stuff ..."
+								bool skip = false;
+								for (auto otherdir : gamedirs) {
+									if (std::filesystem::equivalent(otherdir, gamedir + rootdir + "/" + path)) {
+										skip = true;
+										break;
+									}
+								}
+								if (skip) continue;
+							}
+							sorted.insert(path);
+						}
+					} catch (std::system_error &e) {
+					}
 				}
-			} catch (std::system_error &e) {
 			}
 		}
 

src/Features/AutoSubmit.cpp

@@ -4,6 +4,7 @@
 #include "Features/Hud/Toasts.hpp"
 #include "Features/NetMessage.hpp"
 #include "Modules/Engine.hpp"
+#include "Modules/FileSystem.hpp"
 #include "Modules/Server.hpp"
 #include "Utils/json11.hpp"
 #include <cctype>
@@ -371,7 +372,8 @@ static void submitTime(int score, std::string demopath, bool coop, const char *m
 }
 
 static void loadApiKey(bool output_nonexist) {
-	if (!std::filesystem::exists(API_KEY_FILE)) {
+	auto filepath = fileSystem->FindFileSomewhere(API_KEY_FILE);
+	if (!filepath.has_value()) {
 		if (output_nonexist) {
 			console->Print("API key file " API_KEY_FILE " does not exist!\n");
 		}
@@ -380,7 +382,7 @@ static void loadApiKey(bool output_nonexist) {
 
 	std::string key;
 	{
-		std::ifstream f(API_KEY_FILE);
+		std::ifstream f(filepath.value());
 		std::stringstream buf;
 		buf << f.rdbuf();
 		key = buf.str();

src/Features/Camera.cpp

@@ -742,12 +742,9 @@ CON_COMMAND(sar_cam_path_export,
 	if (rate <= 0) rate = 60; 
 
 	// check if file exists before writing
-	std::ifstream testFile(filename.c_str());
-	if (testFile.good()) {
-		testFile.close();
+	if (std::filesystem::exists(filename)) {
 		return console->Print("File \"%s\" exists and cannot be overwritten.\n", filename.c_str());
 	}
-	testFile.close();
 
 	std::ofstream file(filename.c_str());
 

src/Features/ClassDumper.cpp

@@ -3,6 +3,7 @@
 #include "Modules/Client.hpp"
 #include "Modules/Console.hpp"
 #include "Modules/Engine.hpp"
+#include "Modules/FileSystem.hpp"
 #include "Modules/Server.hpp"
 #include "SAR.hpp"
 #include "Utils/SDK.hpp"
@@ -24,7 +25,8 @@ ClassDumper::ClassDumper()
 void ClassDumper::Dump(bool dumpServer) {
 	auto source = (dumpServer) ? &this->serverClassesFile : &this->clientClassesFile;
 
-	std::ofstream file(*source, std::ios::out | std::ios::trunc);
+	auto filepath = fileSystem->FindFileSomewhere(*source).value_or(*source);
+	std::ofstream file(filepath, std::ios::out | std::ios::trunc);
 	if (!file.good()) {
 		console->Warning("Failed to create file!\n");
 		return file.close();

src/Features/ConfigPlus.cpp

@@ -25,9 +25,11 @@
 
 static std::map<std::string, std::string> g_svars;
 static std::unordered_set<std::string> g_persistentSvars;
+static std::string g_persistentSvarsFile;
 
 ON_INIT {
-	std::ifstream file(PERSISTENT_SVAR_FILENAME);
+	g_persistentSvarsFile = fileSystem->FindFileSomewhere(PERSISTENT_SVAR_FILENAME).value_or(PERSISTENT_SVAR_FILENAME);
+	std::ifstream file(g_persistentSvarsFile.c_str());
 
 	std::string line;
 	std::getline(file, line);
@@ -46,7 +48,7 @@ ON_INIT {
 }
 
 static void SavePersistentSvars() {
-	FILE *fp = fopen(PERSISTENT_SVAR_FILENAME, "w");
+	FILE *fp = fopen(g_persistentSvarsFile.c_str(), "w");
 	if (fp) {
 		for (auto &name : g_persistentSvars) {
 			auto val = g_svars[name];
@@ -68,12 +70,7 @@ CON_COMMAND_F(sar_download_file, "sar_download_file <url> <filepath> [directory]
 	std::string filepath = args[2];
 	std::string gamedir = args.ArgC() > 3 ? args[3] : "";
 
-	auto result = fileSystem->FindFileSomewhere(filepath, gamedir);
-	if (result.has_value()) {
-		filepath = result.value();
-	} else {
-		filepath = std::string(engine->GetGameDirectory()) + "/" + filepath;
-	}
+	filepath = fileSystem->FindFileSomewhere(filepath, gamedir).value_or(std::string(engine->GetGameDirectory()) + "/" + filepath);
 
 	CURL *curl = curl_easy_init();
 	FILE *fp;

src/Features/DataMapDumper.cpp

@@ -2,6 +2,7 @@
 
 #include "Modules/Client.hpp"
 #include "Modules/Console.hpp"
+#include "Modules/FileSystem.hpp"
 #include "Modules/Server.hpp"
 #include "SAR.hpp"
 #include "Utils/Memory.hpp"
@@ -37,7 +38,8 @@ DataMapDumper::DataMapDumper()
 void DataMapDumper::Dump(bool dumpServer) {
 	auto source = (dumpServer) ? &this->serverDataMapFile : &this->clientDataMapFile;
 
-	std::ofstream file(*source, std::ios::out | std::ios::trunc);
+	auto filepath = fileSystem->FindFileSomewhere(*source).value_or(*source);
+	std::ofstream file(filepath, std::ios::out | std::ios::trunc);
 	if (!file.good()) {
 		console->Warning("Failed to create file!\n");
 		return file.close();

src/Features/Demo/DemoGhostPlayer.cpp

@@ -6,6 +6,7 @@
 #include "Features/Session.hpp"
 #include "Modules/Client.hpp"
 #include "Modules/Engine.hpp"
+#include "Modules/FileSystem.hpp"
 #include "Modules/Server.hpp"
 #include "NetworkGhostPlayer.hpp"
 #include "Utils.hpp"
@@ -199,18 +200,19 @@ std::string DemoGhostPlayer::CustomDataToString(std::optional<int> slot) {
 	return Utils::ssprintf("%d", slot);
 }
 
-DECL_COMMAND_FILE_COMPLETION(ghost_set_demo, ".dem", engine->GetGameDirectory(), 1);
+DECL_COMMAND_FILE_COMPLETION(ghost_set_demo, ".dem", "", 1);
 CON_COMMAND_F_COMPLETION(ghost_set_demo, "ghost_set_demo <demo> [ID] - ghost will use this demo. If ID is specified, will create or modify the ID-th ghost\n", 0, AUTOCOMPLETION_FUNCTION(ghost_set_demo)) {
 	if (args.ArgC() < 2) {
 		return console->Print(ghost_set_demo.ThisPtr()->m_pszHelpString);
 	}
 
 	sf::Uint32 ID = args.ArgC() > 2 ? std::atoi(args[2]) : 0;
 	demoGhostPlayer.DeleteGhostsByID(ID);
-	if (demoGhostPlayer.SetupGhostFromDemo(engine->GetGameDirectory() + std::string("/") + args[1], ID, false)) {
+	auto filepath = fileSystem->FindFileSomewhere(args[1]).value_or(engine->GetGameDirectory() + std::string("/") + args[1]);
+	if (demoGhostPlayer.SetupGhostFromDemo(filepath, ID, false)) {
 		console->Print("Ghost successfully created! Final time of the ghost: %s\n", SpeedrunTimer::Format(demoGhostPlayer.GetGhostByID(ID)->GetTotalTime()).c_str());
 	} else {
-		console->Print("Could not parse \"%s\"!\n", (engine->GetGameDirectory() + std::string("/") + args[1]).c_str());
+		console->Print("Could not parse \"%s\"!\n", args[1]);
 	}
 
 	demoGhostPlayer.UpdateGhostsSameMap();
@@ -232,20 +234,20 @@ CON_COMMAND_F_COMPLETION(ghost_set_demos,
 	sf::Uint32 ID = args.ArgC() > 3 ? std::atoi(args[3]) : 0;
 	demoGhostPlayer.DeleteGhostsByID(ID);
 
-	auto dir = engine->GetGameDirectory() + std::string("/") + args[1];
+	auto filepath = fileSystem->FindFileSomewhere(args[1]).value_or(engine->GetGameDirectory() + std::string("/") + args[1]);
 	int counter = firstDemoId > 1 ? firstDemoId : 2;
 
 	bool ok = true;
 
 	if (firstDemoId < 2) {
-		ok = std::filesystem::exists(dir + ".dem");
-		if (!ok || !demoGhostPlayer.SetupGhostFromDemo(dir, ID, true)) {
-			return console->Print("Could not parse \"%s\"!\n", (engine->GetGameDirectory() + std::string("/") + args[1]).c_str());
+		ok = std::filesystem::exists(filepath + ".dem");
+		if (!ok || !demoGhostPlayer.SetupGhostFromDemo(filepath, ID, true)) {
+			return console->Print("Could not parse \"%s\"!\n", filepath.c_str());
 		}
 	}
 
 	while (ok) {
-		auto tmp_dir = dir + "_" + std::to_string(counter) + ".dem";
+		auto tmp_dir = filepath + "_" + std::to_string(counter) + ".dem";
 		ok = std::filesystem::exists(tmp_dir);
 		if (ok && !demoGhostPlayer.SetupGhostFromDemo(tmp_dir, ID, true)) {
 			return console->Print("Could not parse \"%s\"!\n", tmp_dir.c_str());

src/Features/Demo/DemoParser.cpp

@@ -6,6 +6,7 @@
 #include "Features/Hud/Hud.hpp"
 #include "Modules/Console.hpp"
 #include "Modules/Engine.hpp"
+#include "Modules/FileSystem.hpp"
 #include "Variable.hpp"
 
 #include <fstream>
@@ -303,7 +304,7 @@ bool DemoParser::Parse(std::string filePath, Demo *demo, bool ghostRequest, std:
 
 // Commands
 
-DECL_COMMAND_FILE_COMPLETION(sar_time_demo, ".dem", engine->GetGameDirectory(), 1);
+DECL_COMMAND_FILE_COMPLETION(sar_time_demo, ".dem", "", 1);
 CON_COMMAND_F_COMPLETION(sar_time_demo, "sar_time_demo <demo_name> - parses a demo and prints some information about it\n", 0, AUTOCOMPLETION_FUNCTION(sar_time_demo)) {
 	if (args.ArgC() != 2) {
 		return console->Print(sar_time_demo.ThisPtr()->m_pszHelpString);
@@ -324,7 +325,7 @@ CON_COMMAND_F_COMPLETION(sar_time_demo, "sar_time_demo <demo_name> - parses a de
 	parser.outputMode = sar_time_demo_dev.GetInt();
 
 	Demo demo;
-	auto dir = std::string(engine->GetGameDirectory()) + std::string("/") + name;
+	auto dir = fileSystem->FindFileSomewhere(name).value_or(name);
 	if (parser.Parse(dir, &demo)) {
 		parser.Adjust(&demo);
 		console->Print("Demo:     %s\n", name.c_str());
@@ -351,12 +352,12 @@ CON_COMMAND_F_COMPLETION(sar_time_demos, "sar_time_demos <demo_name> [demo_name2
 	parser.outputMode = sar_time_demo_dev.GetInt();
 
 	auto name = std::string();
-	auto dir = std::string(engine->GetGameDirectory()) + std::string("/");
 	for (auto i = 1; i < args.ArgC(); ++i) {
 		name = std::string(args[i]);
 
 		Demo demo;
-		if (parser.Parse(dir + name, &demo)) {
+		auto filepath = fileSystem->FindFileSomewhere(name).value_or(name);
+		if (parser.Parse(filepath, &demo)) {
 			parser.Adjust(&demo);
 			console->Print("Demo:     %s\n", name.c_str());
 			console->Print("Client:   %s\n", demo.clientName);