add markdown support

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "libs/crow"]
 	path = libs/crow
 	url = https://github.com/CrowCpp/Crow
+[submodule "libs/md4c"]
+	path = libs/md4c
+	url = https://github.com/mity/md4c.git

CMakeLists.txt

@@ -33,6 +33,9 @@ else ()
 endif ()
 
 add_subdirectory(libs/BLT)
+
+add_subdirectory(libs/md4c)
+
 include_directories(include/)
 include_directories(${CURL_INCLUDE_DIRS})
 include_directories(${SQLite3_INCLUDE_DIRS})
@@ -49,6 +52,8 @@ target_link_libraries(crowsite Crow::Crow)
 target_link_libraries(crowsite ${CURL_LIBRARIES})
 target_link_libraries(crowsite OpenSSL::SSL OpenSSL::Crypto)
 target_link_libraries(crowsite SQLite::SQLite3)
+target_link_libraries(crowsite md4c-html md4c)
+target_include_directories(crowsite PRIVATE libs/md4c/src)
 target_compile_options(crowsite PRIVATE -Wall -Wextra -Wpedantic)
 
 if (${ENABLE_ADDRSAN} MATCHES ON)

crow_test/webcontent/menu_bar.part

@@ -1,7 +1,7 @@
 <ul class="menu_bar">
     <li class="left"><a id="home" href="/">Home</a></li>
+    <li class="left"><a id="blog" href="/blog/">Blog</a></li>
     <li class="left"><a id="projects" href="/projects/">Projects</a></li>
-    <li class="left"><a id="research" href="/research/">Research</a></li>
     <li class="left"><a id="git" href="/git.html">Git</a></li>
     <li class="left"><a id='about' href="/about.html">About</a></li>
     {{%_logged_in}}
@@ -18,8 +18,10 @@
     const ids = {
         "/": "home",
         "/index.html": "home",
-        "/projects.html": "projects",
-        "/research.html": "research",
+        "/projects/": "projects",
+        "/projects/index.html": "projects",
+        "/blog/": "blog",
+        "/blog/index.html": "blog",
         "/git.html": "git",
         "/about.html": "about",
         "/login.html": "login"

include/crowsite/site/posts.h

@@ -5,8 +5,8 @@
  * See LICENSE file for license detail
  */
 
-#ifndef CROWSITE_PROJECTS_H
-#define CROWSITE_PROJECTS_H
+#ifndef CROWSITE_POSTS_H
+#define CROWSITE_POSTS_H
 
 #include <crowsite/site/cache.h>
 #include <crow/http_request.h>
@@ -26,4 +26,4 @@ namespace cs
 
 }
 
-#endif //CROWSITE_PROJECTS_H
+#endif //CROWSITE_POSTS_H

include/crowsite/util/md_to_html.h

@@ -0,0 +1,18 @@
+#pragma once
+/*
+ * Created by Brett on 27/08/23.
+ * Licensed under GNU General Public License V3.0
+ * See LICENSE file for license detail
+ */
+
+#ifndef CROWSITE_MD_TO_HTML_H
+#define CROWSITE_MD_TO_HTML_H
+
+#include <string>
+
+namespace cs
+{
+    std::string loadMarkdownAsHTML(const std::string& path);
+}
+
+#endif //CROWSITE_MD_TO_HTML_H

include/crowsite/util/memory_reader.h

@@ -0,0 +1,104 @@
+#pragma once
+/*
+ * Created by Brett on 25/08/23.
+ * Licensed under GNU General Public License V3.0
+ * See LICENSE file for license detail
+ */
+
+#ifndef CROWSITE_MEMORY_READER_H
+#define CROWSITE_MEMORY_READER_H
+
+#include <blt/std/filesystem.h>
+#include <utility>
+#include <queue>
+
+namespace cs
+{
+    
+    class memory_reader : public blt::fs::block_reader
+    {
+        private:
+            char* memory;
+            size_t size;
+            size_t read_point = 0;
+            size_t read_amount = 0;
+        public:
+            memory_reader(char* mem, size_t s): block_reader(s), memory(mem), size(s)
+            {}
+            
+            inline void reset()
+            {
+                read_point = 0;
+                read_amount = 0;
+            }
+            
+            int read(char* buffer, size_t bytes) final;
+            
+            inline size_t gcount() final
+            {
+                return read_amount;
+            }
+    };
+    
+    class zip_reader : public blt::fs::block_reader
+    {
+        private:
+            constexpr static size_t BUFFER_SIZE = 1 * 1024 * 1024;
+            
+            struct file
+            {
+                std::string path;
+                size_t file_size;
+            };
+            
+            std::queue<file> files;
+            
+            std::string path;
+            char* zip_buffer;
+            
+            size_t read_point = 0;
+            size_t write_point = 0;
+            size_t read_amount = 0;
+            
+            void calculateFilesystem();
+        
+        public:
+            /**
+             * Creates a zip'd reader from a file or directory. Note: this will read the file or directory as a uncompressed zip file
+             * TODO: Compress
+             * @param path path to file or directory to stream zip
+             */
+            explicit zip_reader(std::string path): block_reader(32 * 1024), path(std::move(path)), zip_buffer(new char[BUFFER_SIZE])
+            {
+                // TODO: all of this later
+                // https://en.wikipedia.org/wiki/ZIP_%28file_format%29
+                // write the zip magic number
+                zip_buffer[write_point++] = 'P';
+                zip_buffer[write_point++] = 'K';
+                zip_buffer[write_point++] = 0x03;
+                zip_buffer[write_point++] = 0x04;
+                // zip version
+                zip_buffer[write_point++] = 0x03;
+                zip_buffer[write_point++] = 0x14;
+                // bit flag
+                zip_buffer[write_point++] = 0;
+                zip_buffer[write_point++] = 0;
+                // compression method
+                zip_buffer[write_point++] = 0;
+                zip_buffer[write_point++] = 0;
+                calculateFilesystem();
+            }
+            
+            int read(char* buffer, size_t bytes) final;
+            
+            inline size_t gcount() final
+            {
+                return read_amount;
+            }
+            
+            ~zip_reader();
+    };
+    
+}
+
+#endif //CROWSITE_MEMORY_READER_H

include/crowsite/utility.h

@@ -30,6 +30,7 @@ namespace cs {
         
         std::string createStaticFilePath(const std::string& file);
         std::string createWebFilePath(const std::string& file);
+        std::string createDataFilePath(const std::string& file);
         
     }
     

libs/crow/CMakeLists.txt

@@ -55,6 +55,13 @@ target_include_directories(Crow
 		$<INSTALL_INTERFACE:include>
 )
 
+#file(GLOB_RECURSE Crow_Header_Files RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} include/*.h)
+##message("Headers: ${Crow_Header_Files}")
+#target_precompile_headers(Crow
+#		INTERFACE
+#		$<BUILD_INTERFACE:${Crow_Header_Files}>
+#)
+
 file(GLOB_RECURSE Crow_Source_Files ${CMAKE_CURRENT_SOURCE_DIR}/src/*.cpp)
 target_sources(Crow INTERFACE ${Crow_Source_Files})
 

libs/crow/include/crow/http_connection.h

@@ -414,6 +414,7 @@ namespace crow
             }
             if (close_connection_)
             {
+                std::cout << "closing connection!\n";
                 adaptor_.shutdown_readwrite();
                 adaptor_.close();
                 CROW_LOG_DEBUG << this << " from write (static)";
@@ -552,7 +553,6 @@ namespace crow
 
         inline void do_write_sync(std::vector<asio::const_buffer>& buffers)
         {
-
             asio::write(adaptor_.socket(), buffers, [&](asio::error_code ec, std::size_t) {
                 if (!ec)
                 {

libs/crow/include/crow/http_response.h

@@ -20,6 +20,7 @@
 #include "crow/logging.h"
 #include "crow/mime_types.h"
 #include "crow/returnable.h"
+#include <blt/std/filesystem.h>
 
 
 namespace crow
@@ -202,7 +203,7 @@ namespace crow
         /// Check whether the response has a static file defined.
         inline bool is_static_type()
         {
-            return file_info.path.size();
+            return !file_info.path.empty() || file_info.reader != nullptr;
         }
 
         /// This constains metadata (coming from the `stat` command) related to any static files associated with this response.
@@ -214,12 +215,22 @@ namespace crow
             std::string path = "";
             struct stat statbuf;
             int statResult;
-            std::ifstream* reader = nullptr;
+            blt::fs::block_reader* reader = nullptr;
         };
 
         /// Return a static file as the response body
         void set_static_file_info(std::string path);
         
+        void set_static_file_reader(blt::fs::block_reader* reader, size_t total_size)
+        {
+            compressed = false;
+            code = 200;
+            file_info.reader = reader;
+            file_info.statbuf.st_size = static_cast<long>(total_size);
+            this->add_header("Content-Length", std::to_string(total_size));
+            this->add_header("Content-Type", "application/octet-stream");
+        }
+
         /// Return a static file as the response body without sanitizing the path (use set_static_file_info instead)
         void set_static_file_info_unsafe(std::string path);
 

libs/crow/src/http_response.cpp

@@ -12,7 +12,7 @@ namespace crow
 #endif
         if (file_info.statResult == 0 && S_ISREG(file_info.statbuf.st_mode))
         {
-            std::size_t last_dot = path.find_last_of(".");
+            std::size_t last_dot = path.find_last_of('.');
             std::string extension = path.substr(last_dot + 1);
             code = 200;
             this->add_header("Content-Length", std::to_string(file_info.statbuf.st_size));

libs/crow/tests/catch.hpp

@@ -8476,16 +8476,16 @@ namespace clara {
 namespace TextFlow {
 
 inline auto isWhitespace(char c) -> bool {
-	static std::string chars = " \t\n\r";
-	return chars.find(c) != std::string::npos;
+	static std::string buffer = " \t\n\r";
+	return buffer.find(c) != std::string::npos;
 }
 inline auto isBreakableBefore(char c) -> bool {
-	static std::string chars = "[({<|";
-	return chars.find(c) != std::string::npos;
+	static std::string buffer = "[({<|";
+	return buffer.find(c) != std::string::npos;
 }
 inline auto isBreakableAfter(char c) -> bool {
-	static std::string chars = "])}>.,:;*+-=&/\\";
-	return chars.find(c) != std::string::npos;
+	static std::string buffer = "])}>.,:;*+-=&/\\";
+	return buffer.find(c) != std::string::npos;
 }
 
 class Columns;

libs/md4c

@@ -0,0 +1 @@
+Subproject commit e9ff661ff818ee94a4a231958d9b6768dc6882c9

src/crowsite/site/posts.cpp

@@ -0,0 +1,27 @@
+/*
+ * Created by Brett on 23/08/23.
+ * Licensed under GNU General Public License V3.0
+ * See LICENSE file for license detail
+ */
+#include <crowsite/site/posts.h>
+#include <crowsite/utility.h>
+#include <crowsite/util/md_to_html.h>
+#include <blt/std/logging.h>
+
+namespace cs
+{
+    
+    crow::response handleProjectPage(const request_info& req)
+    {
+        std::string buffer;
+        buffer += "<html><head></head><body>";
+        auto htmlData = loadMarkdownAsHTML(cs::fs::createDataFilePath("Billionaire Propaganda.md"));
+        buffer += htmlData;
+        buffer += "</body>";
+        
+        return buffer;
+    }
+    
+    
+    
+}

src/crowsite/site/projects.cpp

@@ -1,17 +0,0 @@
-/*
- * Created by Brett on 23/08/23.
- * Licensed under GNU General Public License V3.0
- * See LICENSE file for license detail
- */
-#include <crowsite/site/projects.h>
-
-namespace cs
-{
-    
-    crow::response handleProjectPage(const request_info& req)
-    {
-        
-        
-        return crow::response("hel funny");
-    }
-}

src/crowsite/util/md_to_html.cpp

@@ -0,0 +1,76 @@
+/*
+ * Created by Brett on 27/08/23.
+ * Licensed under GNU General Public License V3.0
+ * See LICENSE file for license detail
+ */
+#include <crowsite/util/md_to_html.h>
+#include <md4c-html.h>
+#include <sstream>
+#include <fstream>
+#include <cstring>
+
+namespace cs {
+    
+    class string_buffer
+    {
+        private:
+            constexpr static size_t initial_size = 512;
+            char* buffer;
+            size_t buffer_size = initial_size;
+            size_t used_size = 0;
+            
+            void expand()
+            {
+                size_t new_size = buffer_size * 2;
+                char* tmp = new char[new_size];
+                for (size_t i = 0; i < used_size; i++)
+                    tmp[i] = buffer[i];
+                delete[] buffer;
+                buffer = tmp;
+                buffer_size = new_size;
+            }
+        
+        public:
+            string_buffer(): buffer(new char[initial_size])
+            {}
+            
+            ~string_buffer()
+            { delete[] buffer; }
+            
+            void append(const MD_CHAR* text, MD_SIZE size)
+            {
+                if (used_size + size >= buffer_size)
+                    expand();
+                std::memcpy(&buffer[used_size], text, size);
+                used_size += size;
+            }
+            
+            std::string str()
+            {
+                std::string out (buffer, buffer + used_size);
+                return out;
+            }
+    };
+    
+    void process_output(const MD_CHAR* text, MD_SIZE size, void* data)
+    {
+        reinterpret_cast<string_buffer*>(data)->append(text, size);
+    }
+    
+    std::string loadMarkdownAsHTML(const std::string& path)
+    {
+        std::string mdData;
+        string_buffer output;
+        {
+            std::ifstream file(path);
+            std::stringstream stream;
+            stream << file.rdbuf();
+            mdData = stream.str();
+        }
+        const unsigned int parse_flags =
+                MD_FLAG_TABLES | MD_FLAG_TASKLISTS | MD_FLAG_STRIKETHROUGH | MD_FLAG_PERMISSIVEURLAUTOLINKS | MD_FLAG_PERMISSIVEEMAILAUTOLINKS
+                | MD_FLAG_PERMISSIVEWWWAUTOLINKS | MD_FLAG_LATEXMATHSPANS | MD_FLAG_WIKILINKS | MD_FLAG_UNDERLINE;
+        md_html(mdData.c_str(), mdData.size(), process_output, &output, parse_flags, 0);
+        return output.str();
+    }
+}

src/crowsite/util/memory_reader.cpp

@@ -0,0 +1,70 @@
+/*
+ * Created by Brett on 25/08/23.
+ * Licensed under GNU General Public License V3.0
+ * See LICENSE file for license detail
+ */
+#include <crowsite/util/memory_reader.h>
+
+#include <cstring>
+#include <blt/std/logging.h>
+#include <zlib.h>
+#include <filesystem>
+
+namespace cs
+{
+    
+    int memory_reader::read(char* buffer, size_t bytes)
+    {
+        auto end_point = read_point + bytes;
+        if (end_point <= size)
+        {
+            read_amount = bytes;
+            std::memcpy(buffer, &memory[read_point], read_amount);
+        } else
+        {
+            //BLT_INFO("Data: EP: %d, RP: %d, BTs: %d, SZ: %d", end_point, read_point, bytes, size);
+            // can only read this much data!
+            read_amount = size - read_point;
+            if (read_amount == 0)
+                return 0;
+            std::memcpy(buffer, &memory[read_point], read_amount);
+        }
+        read_point = end_point;
+        return 0;
+    }
+    
+    int zip_reader::read(char*, size_t)
+    {
+        return 0;
+    }
+    
+    zip_reader::~zip_reader()
+    {
+        delete[] zip_buffer;
+    }
+    
+    void zip_reader::calculateFilesystem()
+    {
+        if (!std::filesystem::is_directory(path))
+        {
+            files.push({path, std::filesystem::file_size(path)});
+            return;
+        }
+        std::queue<std::string> directories;
+        directories.push(path);
+        while (!directories.empty())
+        {
+            auto front = directories.front();
+            directories.pop();
+            
+            std::filesystem::directory_iterator dir(front);
+            for (const auto& p : dir)
+            {
+                if (p.is_directory())
+                    directories.push(p.path());
+                else
+                    files.push({p.path(), p.file_size()});
+            }
+        }
+    }
+}

src/crowsite/utility.cpp

@@ -76,6 +76,18 @@ namespace cs
                 throw std::runtime_error("Unable to create file path because file does not exist!");
             return path;
         }
+        
+        std::string createDataFilePath(const std::string& file)
+        {
+            auto path = std::string(SITE_FILES_PATH);
+            if (!path.ends_with('/'))
+                path += '/';
+            path += "data/";
+            path += file;
+            if (!std::filesystem::exists(path))
+                throw std::runtime_error("Unable to create file path because file does not exist!");
+            return path;
+        }
     }
     
 }

src/main.cpp

@@ -10,7 +10,8 @@
 #include <crowsite/site/auth.h>
 #include <crow/middlewares/session.h>
 #include <crow/middlewares/cookie_parser.h>
-#include <crowsite/site/projects.h>
+#include <crowsite/site/posts.h>
+#include <crowsite/util/memory_reader.h>
 
 using Session = crow::SessionMiddleware<crow::FileStore>;
 using CrowApp = crow::App<crow::CookieParser, Session>;
@@ -20,7 +21,7 @@ class BLT_CrowLogger : public crow::ILogHandler
     public:
         void log(std::string message, crow::LogLevel crow_level) final
         {
-            blt::logging::log_level blt_level;
+            blt::logging::log_level blt_level = blt::logging::log_level::NONE;
             switch (crow_level)
             {
                 case crow::LogLevel::DEBUG:
@@ -100,7 +101,8 @@ bool isUserAdmin(CrowApp& app, const crow::request& req)
     return cs::isUserAdmin(cs::getUserFromID(s_clientID));
 }
 
-void generateRuntimeContext(const site_params& params, cs::RuntimeContext& context){
+void generateRuntimeContext(const site_params& params, cs::RuntimeContext& context)
+{
     auto& session = params.app.get_context<Session>(params.req);
     auto s_clientID = session.get("clientID", "");
     auto s_clientToken = session.get("clientToken", "");
@@ -162,7 +164,8 @@ crow::response handle_root_page(const site_params& params)
         if (cs::isUserLoggedIn(s_clientID, s_clientToken))
         {
             ctx["_logged_in"] = true;
-        } else {
+        } else
+        {
             ctx["_not_logged_in"] = true;
         }
         
@@ -183,7 +186,6 @@ crow::response handle_auth_page(const site_params& params)
         return cs::redirect("/login.html");
     
     
-    
     return handle_root_page(params);
 }