From 51eb1711b9b6008a0592b034fe92da91d280cd41 Mon Sep 17 00:00:00 2001 From: Brett Date: Sun, 6 Apr 2025 21:27:44 -0400 Subject: [PATCH 01/32] why --- CMakeLists.txt | 3 ++- default.nix | 36 ++++++++++++++++++++++++++++++ include/blt/gp/defines.h | 48 ++++++++++++++++++++++++++++++++++++++++ include/blt/gp/fwdecl.h | 2 +- lib/blt | 2 +- src/transformers.cpp | 1 + 6 files changed, 89 insertions(+), 3 deletions(-) create mode 100644 default.nix create mode 100644 include/blt/gp/defines.h diff --git a/CMakeLists.txt b/CMakeLists.txt index bd682e0..b839fcc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,7 +37,8 @@ option(ENABLE_TSAN "Enable the thread data race sanitizer" OFF) option(BUILD_EXAMPLES "Build example programs. This will build with CTest" OFF) option(BUILD_GP_TESTS "Build test programs." OFF) option(DEBUG_LEVEL "Enable debug features which prints extra information to the console, might slow processing down. [0, 3)" 0) -option(TRACK_ALLOCATIONS "Track total allocations. Can be accessed with blt::gp::tracker" OFF) +option(BLT_GP_DEBUG_CHECK_TREES "Enable checking of trees after every operation" OFF) +option(BLT_GP_DEBUG_TRACK_ALLOCATIONS "Track total allocations. Can be accessed with blt::gp::tracker" OFF) set(CMAKE_CXX_STANDARD 17) diff --git a/default.nix b/default.nix new file mode 100644 index 0000000..9d20b5e --- /dev/null +++ b/default.nix @@ -0,0 +1,36 @@ +{ pkgs ? (import { + config.allowUnfree = true; + config.segger-jlink.acceptLicense = true; +}), ... }: +pkgs.mkShell +{ + buildInputs = with pkgs; [ + cmake + gcc + clang + emscripten + ninja + renderdoc + valgrind + opentelemetry-cpp + opentelemetry-cpp.dev + ]; + nativeBuildInputs = with pkgs; [ + pkg-config + abseil-cpp + opentelemetry-cpp + opentelemetry-cpp.dev + ]; + propagatedBuildInputs = with pkgs; [ + gtest + gtest.dev + grpc + protobuf + curl + abseil-cpp + opentelemetry-cpp + opentelemetry-cpp.dev + opentelemetry-collector + ]; + LD_LIBRARY_PATH="/run/opengl-driver/lib:/run/opengl-driver-32/lib"; +} diff --git a/include/blt/gp/defines.h b/include/blt/gp/defines.h new file mode 100644 index 0000000..80b527a --- /dev/null +++ b/include/blt/gp/defines.h @@ -0,0 +1,48 @@ +#pragma once +/* + * Copyright (C) 2024 Brett Terpstra + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef BLT_GP_DEFINES_H +#define BLT_GP_DEFINES_H + +#include + +#if BLT_DEBUG_LEVEL > 0 + #if defined(__has_include) &&__has_include() + #define BLT_DEBUG_OTEL_ENABLED 1 + #endif +#endif + +#if BLT_DEBUG_LEVEL > 1 + #define BLT_GP_DEBUG_TRACK_ALLOCATIONS +#endif + +#if BLT_DEBUG_LEVEL > 2 + #define BLT_GP_DEBUG_CHECK_TREES +#endif + +#ifdef BLT_GP_DEBUG_TRACK_ALLOCATIONS + #undef BLT_GP_DEBUG_TRACK_ALLOCATIONS + #define BLT_GP_DEBUG_TRACK_ALLOCATIONS +#endif + +#ifdef BLT_GP_DEBUG_CHECK_TREES + #undef BLT_GP_DEBUG_CHECK_TREES + #define BLT_GP_DEBUG_CHECK_TREES 1 +#endif + +#endif //BLT_GP_DEFINES_H diff --git a/include/blt/gp/fwdecl.h b/include/blt/gp/fwdecl.h index e13bec7..66cef46 100644 --- a/include/blt/gp/fwdecl.h +++ b/include/blt/gp/fwdecl.h @@ -20,7 +20,7 @@ #define BLT_GP_FWDECL_H #include -#include +#include #include #include #include diff --git a/lib/blt b/lib/blt index baa5952..3cdceda 160000 --- a/lib/blt +++ b/lib/blt @@ -1 +1 @@ -Subproject commit baa5952666594ce0d07a2b013e46c4bc343ba164 +Subproject commit 3cdceda227883bfd1bf8c41b466ef94c8271b501 diff --git a/src/transformers.cpp b/src/transformers.cpp index 511ad74..de7c1e7 100644 --- a/src/transformers.cpp +++ b/src/transformers.cpp @@ -24,6 +24,7 @@ #include #include + namespace blt::gp { #if BLT_DEBUG_LEVEL >= 2 || defined(BLT_TRACK_ALLOCATIONS) From ece76e1a91f0ad9d630c4092c8e92ea9c1f73bbc Mon Sep 17 00:00:00 2001 From: Brett Date: Sun, 6 Apr 2025 22:05:06 -0400 Subject: [PATCH 02/32] puppy --- .idea/vcs.xml | 110 +++++++++++++++++++++++++++++++++++++++++++++++++ CMakeLists.txt | 2 +- default.nix | 2 + lib/blt | 2 +- 4 files changed, 114 insertions(+), 2 deletions(-) diff --git a/.idea/vcs.xml b/.idea/vcs.xml index 0a294a1..90587b6 100644 --- a/.idea/vcs.xml +++ b/.idea/vcs.xml @@ -2,6 +2,116 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/CMakeLists.txt b/CMakeLists.txt index 97ee352..26048fc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -27,7 +27,7 @@ macro(compile_options target_name) sanitizers(${target_name}) endmacro() -project(blt-gp VERSION 0.4.9) +project(blt-gp VERSION 0.4.10) include(CTest) diff --git a/default.nix b/default.nix index 9d20b5e..dd3cde3 100644 --- a/default.nix +++ b/default.nix @@ -31,6 +31,8 @@ pkgs.mkShell opentelemetry-cpp opentelemetry-cpp.dev opentelemetry-collector + protobufc + protobufc.dev ]; LD_LIBRARY_PATH="/run/opengl-driver/lib:/run/opengl-driver-32/lib"; } diff --git a/lib/blt b/lib/blt index 3cdceda..2d9b96f 160000 --- a/lib/blt +++ b/lib/blt @@ -1 +1 @@ -Subproject commit 3cdceda227883bfd1bf8c41b466ef94c8271b501 +Subproject commit 2d9b96f1155427fdcc0491aadce93394503f6d66 From d14ccfd1da376ad0f202ab56e672764c8938548d Mon Sep 17 00:00:00 2001 From: Brett Date: Tue, 8 Apr 2025 01:50:23 -0400 Subject: [PATCH 03/32] add ability to write trees to file --- .idea/editor.xml | 239 ----------------------------------------- .idea/vcs.xml | 110 ------------------- CMakeLists.txt | 2 +- default.nix | 28 +++-- include/blt/gp/stack.h | 13 +++ include/blt/gp/tree.h | 11 +- lib/blt | 2 +- src/transformers.cpp | 1 - src/tree.cpp | 85 +++++++++++++++ 9 files changed, 127 insertions(+), 364 deletions(-) diff --git a/.idea/editor.xml b/.idea/editor.xml index 04cdbc9..b465c40 100644 --- a/.idea/editor.xml +++ b/.idea/editor.xml @@ -1,245 +1,6 @@ - diff --git a/.idea/vcs.xml b/.idea/vcs.xml index 90587b6..0a294a1 100644 --- a/.idea/vcs.xml +++ b/.idea/vcs.xml @@ -2,116 +2,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/CMakeLists.txt b/CMakeLists.txt index 26048fc..c459ccd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -27,7 +27,7 @@ macro(compile_options target_name) sanitizers(${target_name}) endmacro() -project(blt-gp VERSION 0.4.10) +project(blt-gp VERSION 0.5.0) include(CTest) diff --git a/default.nix b/default.nix index dd3cde3..e91d468 100644 --- a/default.nix +++ b/default.nix @@ -12,27 +12,33 @@ pkgs.mkShell ninja renderdoc valgrind - opentelemetry-cpp - opentelemetry-cpp.dev + gtest + opentelemetry-cpp + opentelemetry-cpp.dev ]; nativeBuildInputs = with pkgs; [ pkg-config - abseil-cpp opentelemetry-cpp opentelemetry-cpp.dev ]; propagatedBuildInputs = with pkgs; [ - gtest - gtest.dev - grpc - protobuf - curl abseil-cpp + protobuf + grpc + prometheus-cpp + prometheus-cpp.dev + openssl + openssl.dev opentelemetry-cpp opentelemetry-cpp.dev - opentelemetry-collector - protobufc - protobufc.dev + civetweb + civetweb.dev + c-ares + c-ares.dev + nlohmann_json + glibc + glibc.dev + curl ]; LD_LIBRARY_PATH="/run/opengl-driver/lib:/run/opengl-driver-32/lib"; } diff --git a/include/blt/gp/stack.h b/include/blt/gp/stack.h index a83457c..e815dcb 100644 --- a/include/blt/gp/stack.h +++ b/include/blt/gp/stack.h @@ -307,6 +307,12 @@ namespace blt::gp expand_raw(bytes); } + void resize(const size_t bytes) + { + reserve(bytes); + bytes_stored = bytes; + } + [[nodiscard]] size_t stored() const { return bytes_stored; @@ -322,6 +328,13 @@ namespace blt::gp bytes_stored = 0; } + [[nodiscard]] auto* data() const + { + return data_; + } + + + private: void expand(const size_t bytes) { diff --git a/include/blt/gp/tree.h b/include/blt/gp/tree.h index 18e75be..b92f5b0 100644 --- a/include/blt/gp/tree.h +++ b/include/blt/gp/tree.h @@ -242,7 +242,6 @@ namespace blt::gp bytes = 0; } } - private: tree_t& tree; u8* data; @@ -608,6 +607,16 @@ namespace blt::gp }; } + [[nodiscard]] size_t required_size() const; + + void to_byte_array(std::byte* out) const; + + void to_file(FILE* file) const; + + void from_byte_array(const std::byte* in); + + void from_file(FILE* file); + ~tree_t() { clear(*m_program); diff --git a/lib/blt b/lib/blt index 2d9b96f..322a533 160000 --- a/lib/blt +++ b/lib/blt @@ -1 +1 @@ -Subproject commit 2d9b96f1155427fdcc0491aadce93394503f6d66 +Subproject commit 322a533fd9412167e216188a4713a5aa4a99c2cd diff --git a/src/transformers.cpp b/src/transformers.cpp index 2b9221d..b7bf704 100644 --- a/src/transformers.cpp +++ b/src/transformers.cpp @@ -24,7 +24,6 @@ #include #include - namespace blt::gp { #if BLT_DEBUG_LEVEL >= 2 || defined(BLT_TRACK_ALLOCATIONS) diff --git a/src/tree.cpp b/src/tree.cpp index 7a45418..a5ebcfc 100644 --- a/src/tree.cpp +++ b/src/tree.cpp @@ -668,6 +668,91 @@ namespace blt::gp return {point, m_program->get_operator_info(operations[point].id()).return_type}; } + size_t tree_t::required_size() const + { + // 2 size_t used to store expected_length of operations + size of the values stack + return 2 * sizeof(size_t) + operations.size() * sizeof(size_t) + values.bytes_in_head(); + } + + void tree_t::to_byte_array(std::byte* out) const + { + const auto op_size = operations.size(); + std::memcpy(out, &op_size, sizeof(size_t)); + out += sizeof(size_t); + for (const auto& op : operations) + { + constexpr auto size_of_op = sizeof(operator_id); + auto id = op.id(); + std::memcpy(out, &id, size_of_op); + out += size_of_op; + } + const auto val_size = values.bytes_in_head(); + std::memcpy(out, &val_size, sizeof(size_t)); + out += sizeof(size_t); + std::memcpy(out, values.data(), val_size); + } + + void tree_t::to_file(FILE* file) const + { + const auto op_size = operations.size(); + BLT_ASSERT(std::fwrite(&op_size, sizeof(size_t), 1, file) == sizeof(size_t)); + for (const auto& op : operations) + { + auto id = op.id(); + std::fwrite(&id, sizeof(operator_id), 1, file); + } + const auto val_size = values.bytes_in_head(); + BLT_ASSERT(std::fwrite(&val_size, sizeof(size_t), 1, file) == sizeof(size_t)); + BLT_ASSERT(std::fwrite(values.data(), val_size, 1, file) == val_size); + } + + void tree_t::from_byte_array(const std::byte* in) + { + size_t ops_to_read; + std::memcpy(&ops_to_read, in, sizeof(size_t)); + in += sizeof(size_t); + operations.reserve(ops_to_read); + for (size_t i = 0; i < ops_to_read; i++) + { + operator_id id; + std::memcpy(&id, in, sizeof(operator_id)); + in += sizeof(operator_id); + operations.push_back({ + m_program->get_typesystem().get_type(m_program->get_operator_info(id).return_type).size(), + id, + m_program->is_operator_ephemeral(id), + m_program->get_operator_flags(id) + }); + } + size_t val_size; + std::memcpy(&val_size, in, sizeof(size_t)); + in += sizeof(size_t); + // TODO replace instances of u8 that are used to alias types with the proper std::byte + values.copy_from(reinterpret_cast(in), val_size); + } + + void tree_t::from_file(FILE* file) + { + size_t ops_to_read; + BLT_ASSERT(std::fread(&ops_to_read, sizeof(size_t), 1, file) == sizeof(size_t)); + operations.reserve(ops_to_read); + for (size_t i = 0; i < ops_to_read; i++) + { + operator_id id; + BLT_ASSERT(std::fread(&id, sizeof(operator_id), 1, file) == sizeof(operator_id)); + operations.push_back({ + m_program->get_typesystem().get_type(m_program->get_operator_info(id).return_type).size(), + id, + m_program->is_operator_ephemeral(id), + m_program->get_operator_flags(id) + }); + } + size_t val_size; + BLT_ASSERT(std::fread(&val_size, sizeof(size_t), 1, file) == sizeof(size_t)); + values.resize(val_size); + BLT_ASSERT(std::fread(values.data(), val_size, 1, file) == val_size); + } + void tree_t::modify_operator(const size_t point, operator_id new_id, std::optional return_type) { if (!return_type) From 62af2584d15bc72ba0b07c71b92a83705e562aa8 Mon Sep 17 00:00:00 2001 From: Brett Date: Fri, 11 Apr 2025 19:49:57 -0400 Subject: [PATCH 04/32] hello --- include/blt/gp/program.h | 52 +++++++++++++++++++++++----------------- include/blt/gp/sync.h | 27 +++++++++++++++++++++ include/blt/gp/tree.h | 5 ++-- lib/blt | 2 +- src/program.cpp | 8 +++++++ src/sync.cpp | 23 ++++++++++++++++++ src/tree.cpp | 20 ++++++++-------- 7 files changed, 102 insertions(+), 35 deletions(-) create mode 100644 include/blt/gp/sync.h create mode 100644 src/sync.cpp diff --git a/include/blt/gp/program.h b/include/blt/gp/program.h index 4183cc9..bf63684 100644 --- a/include/blt/gp/program.h +++ b/include/blt/gp/program.h @@ -640,7 +640,7 @@ namespace blt::gp return thread_helper.lifetime_over; } - operator_id select_terminal(type_id id) + operator_id select_terminal(const type_id id) { // we wanted a terminal, but could not find one, so we will select from a function that has a terminal if (storage.terminals[id].empty()) @@ -648,7 +648,7 @@ namespace blt::gp return get_random().select(storage.terminals[id]); } - operator_id select_non_terminal(type_id id) + operator_id select_non_terminal(const type_id id) { // non-terminal doesn't exist, return a terminal. This is useful for types that are defined only to have a random value, nothing more. // was considering an std::optional<> but that would complicate the generator code considerably. I'll mark this as a TODO for v2 @@ -657,7 +657,7 @@ namespace blt::gp return get_random().select(storage.non_terminals[id]); } - operator_id select_non_terminal_too_deep(type_id id) + operator_id select_non_terminal_too_deep(const type_id id) { // this should probably be an error. if (storage.operators_ordered_terminals[id].empty()) @@ -688,32 +688,32 @@ namespace blt::gp return storage.system; } - [[nodiscard]] operator_info_t& get_operator_info(operator_id id) + [[nodiscard]] operator_info_t& get_operator_info(const operator_id id) { return storage.operators[id]; } - [[nodiscard]] detail::print_func_t& get_print_func(operator_id id) + [[nodiscard]] detail::print_func_t& get_print_func(const operator_id id) { return storage.print_funcs[id]; } - [[nodiscard]] detail::destroy_func_t& get_destroy_func(operator_id id) + [[nodiscard]] detail::destroy_func_t& get_destroy_func(const operator_id id) { return storage.destroy_funcs[id]; } - [[nodiscard]] std::optional get_name(operator_id id) const + [[nodiscard]] std::optional get_name(const operator_id id) const { return storage.names[id]; } - [[nodiscard]] tracked_vector& get_type_terminals(type_id id) + [[nodiscard]] tracked_vector& get_type_terminals(const type_id id) { return storage.terminals[id]; } - [[nodiscard]] tracked_vector& get_type_non_terminals(type_id id) + [[nodiscard]] tracked_vector& get_type_non_terminals(const type_id id) { return storage.non_terminals[id]; } @@ -753,47 +753,55 @@ namespace blt::gp storage = std::move(op); } - template - std::array get_best_indexes() + template + std::array get_best_indexes() { - std::array arr; + std::array arr; - tracked_vector> values; + tracked_vector> values; values.reserve(current_pop.get_individuals().size()); - for (const auto& [index, value] : blt::enumerate(current_pop.get_individuals())) + for (const auto& [index, value] : enumerate(current_pop.get_individuals())) values.emplace_back(index, value.fitness.adjusted_fitness); std::sort(values.begin(), values.end(), [](const auto& a, const auto& b) { return a.second > b.second; }); - for (blt::size_t i = 0; i < std::min(size, config.population_size); i++) + for (size_t i = 0; i < std::min(size, config.population_size); ++i) arr[i] = values[i].first; - for (blt::size_t i = std::min(size, config.population_size); i < size; i++) + for (size_t i = std::min(size, config.population_size); i < size; ++i) arr[i] = 0; return arr; } - template + template auto get_best_trees() { return convert_array, size>>(get_best_indexes(), - [this](auto&& arr, blt::size_t index) -> tree_t& { + [this](auto&& arr, size_t index) -> tree_t& { return current_pop.get_individuals()[arr[index]].tree; - }, std::make_integer_sequence()); + }, std::make_integer_sequence()); } - template + template auto get_best_individuals() { return convert_array, size>>(get_best_indexes(), - [this](auto&& arr, blt::size_t index) -> individual_t& { + [this](auto&& arr, size_t index) -> individual_t& { return current_pop.get_individuals()[arr[index]]; - }, std::make_integer_sequence()); + }, std::make_integer_sequence()); } + void save_generation(fs::writer_t& writer); + + void save_state(fs::writer_t& writer); + + void load_generation(fs::reader_t& reader); + + void load_state(fs::reader_t& reader); + private: template auto single_threaded_fitness_eval() diff --git a/include/blt/gp/sync.h b/include/blt/gp/sync.h new file mode 100644 index 0000000..cc987ed --- /dev/null +++ b/include/blt/gp/sync.h @@ -0,0 +1,27 @@ +#pragma once +/* + * Copyright (C) 2024 Brett Terpstra + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef BLT_GP_SYNC_H +#define BLT_GP_SYNC_H + +namespace blt::gp +{ + +} + +#endif //BLT_GP_SYNC_H diff --git a/include/blt/gp/tree.h b/include/blt/gp/tree.h index b92f5b0..f9a4be5 100644 --- a/include/blt/gp/tree.h +++ b/include/blt/gp/tree.h @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -611,11 +612,11 @@ namespace blt::gp void to_byte_array(std::byte* out) const; - void to_file(FILE* file) const; + void to_file(fs::writer_t& file) const; void from_byte_array(const std::byte* in); - void from_file(FILE* file); + void from_file(fs::reader_t& file); ~tree_t() { diff --git a/lib/blt b/lib/blt index 322a533..78c219c 160000 --- a/lib/blt +++ b/lib/blt @@ -1 +1 @@ -Subproject commit 322a533fd9412167e216188a4713a5aa4a99c2cd +Subproject commit 78c219cc67f1fe6b3c7076e2c727f8f4cfffd859 diff --git a/src/program.cpp b/src/program.cpp index 1cee290..2a0d6dd 100644 --- a/src/program.cpp +++ b/src/program.cpp @@ -56,6 +56,14 @@ namespace blt::gp return allocator; } + void gp_program::save_state(fs::writer_t& writer) + { + } + + void gp_program::load_state(fs::reader_t& reader) + { + } + void gp_program::create_threads() { #ifdef BLT_TRACK_ALLOCATIONS diff --git a/src/sync.cpp b/src/sync.cpp new file mode 100644 index 0000000..88ab06d --- /dev/null +++ b/src/sync.cpp @@ -0,0 +1,23 @@ +/* + * + * Copyright (C) 2025 Brett Terpstra + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#include + +namespace blt::gp +{ + +} \ No newline at end of file diff --git a/src/tree.cpp b/src/tree.cpp index a5ebcfc..5f7a78b 100644 --- a/src/tree.cpp +++ b/src/tree.cpp @@ -692,18 +692,18 @@ namespace blt::gp std::memcpy(out, values.data(), val_size); } - void tree_t::to_file(FILE* file) const + void tree_t::to_file(fs::writer_t& file) const { const auto op_size = operations.size(); - BLT_ASSERT(std::fwrite(&op_size, sizeof(size_t), 1, file) == sizeof(size_t)); + BLT_ASSERT(file.write(&op_size, sizeof(size_t)) == sizeof(size_t)); for (const auto& op : operations) { auto id = op.id(); - std::fwrite(&id, sizeof(operator_id), 1, file); + file.write(&id, sizeof(operator_id)); } const auto val_size = values.bytes_in_head(); - BLT_ASSERT(std::fwrite(&val_size, sizeof(size_t), 1, file) == sizeof(size_t)); - BLT_ASSERT(std::fwrite(values.data(), val_size, 1, file) == val_size); + BLT_ASSERT(file.write(&val_size, sizeof(size_t)) == sizeof(size_t)); + BLT_ASSERT(file.write(values.data(), val_size) == val_size); } void tree_t::from_byte_array(const std::byte* in) @@ -731,15 +731,15 @@ namespace blt::gp values.copy_from(reinterpret_cast(in), val_size); } - void tree_t::from_file(FILE* file) + void tree_t::from_file(fs::reader_t& file) { size_t ops_to_read; - BLT_ASSERT(std::fread(&ops_to_read, sizeof(size_t), 1, file) == sizeof(size_t)); + BLT_ASSERT(file.read(&ops_to_read, sizeof(size_t)) == sizeof(size_t)); operations.reserve(ops_to_read); for (size_t i = 0; i < ops_to_read; i++) { operator_id id; - BLT_ASSERT(std::fread(&id, sizeof(operator_id), 1, file) == sizeof(operator_id)); + BLT_ASSERT(file.read(&id, sizeof(operator_id)) == sizeof(operator_id)); operations.push_back({ m_program->get_typesystem().get_type(m_program->get_operator_info(id).return_type).size(), id, @@ -748,9 +748,9 @@ namespace blt::gp }); } size_t val_size; - BLT_ASSERT(std::fread(&val_size, sizeof(size_t), 1, file) == sizeof(size_t)); + BLT_ASSERT(file.read(&val_size, sizeof(size_t)) == sizeof(size_t)); values.resize(val_size); - BLT_ASSERT(std::fread(values.data(), val_size, 1, file) == val_size); + BLT_ASSERT(file.read(values.data(), val_size) == val_size); } void tree_t::modify_operator(const size_t point, operator_id new_id, std::optional return_type) From fab86f81620dd5a0eab8f82ba39d4fceaf6477d8 Mon Sep 17 00:00:00 2001 From: Brett Laptop Date: Fri, 11 Apr 2025 20:53:40 -0400 Subject: [PATCH 05/32] watching fun --- .idea/editor.xml | 243 +++++++++++++++++++ .idea/inspectionProfiles/Project_Default.xml | 6 + CMakeLists.txt | 2 +- include/blt/gp/fwdecl.h | 4 - include/blt/gp/sync.h | 28 +++ lib/blt | 2 +- src/program.cpp | 8 + src/sync.cpp | 73 +++++- 8 files changed, 359 insertions(+), 7 deletions(-) create mode 100644 .idea/inspectionProfiles/Project_Default.xml diff --git a/.idea/editor.xml b/.idea/editor.xml index b465c40..8abdf90 100644 --- a/.idea/editor.xml +++ b/.idea/editor.xml @@ -1,7 +1,250 @@ + \ No newline at end of file diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml new file mode 100644 index 0000000..8bf9b9f --- /dev/null +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index c459ccd..26bcd99 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -27,7 +27,7 @@ macro(compile_options target_name) sanitizers(${target_name}) endmacro() -project(blt-gp VERSION 0.5.0) +project(blt-gp VERSION 0.5.1) include(CTest) diff --git a/include/blt/gp/fwdecl.h b/include/blt/gp/fwdecl.h index 66cef46..40ee4ad 100644 --- a/include/blt/gp/fwdecl.h +++ b/include/blt/gp/fwdecl.h @@ -23,10 +23,6 @@ #include #include #include -#include -#include -#include -#include #include #include diff --git a/include/blt/gp/sync.h b/include/blt/gp/sync.h index cc987ed..8988b6e 100644 --- a/include/blt/gp/sync.h +++ b/include/blt/gp/sync.h @@ -19,9 +19,37 @@ #ifndef BLT_GP_SYNC_H #define BLT_GP_SYNC_H +#include +#include + namespace blt::gp { + class sync_t + { + public: + explicit sync_t(gp_program& program); + void trigger(u64 current_time); + + sync_t& with_timer(u64 seconds) + { + m_timer_seconds = seconds; + return *this; + } + + sync_t& every_generations(u64 generations) + { + m_generations = generations; + return *this; + } + + ~sync_t(); + + private: + gp_program* m_program; + std::optional m_timer_seconds; + std::optional m_generations; + }; } #endif //BLT_GP_SYNC_H diff --git a/lib/blt b/lib/blt index 78c219c..b6fc170 160000 --- a/lib/blt +++ b/lib/blt @@ -1 +1 @@ -Subproject commit 78c219cc67f1fe6b3c7076e2c727f8f4cfffd859 +Subproject commit b6fc1703995195af611532cdffda52c86993e4ce diff --git a/src/program.cpp b/src/program.cpp index 2a0d6dd..7ac6018 100644 --- a/src/program.cpp +++ b/src/program.cpp @@ -56,6 +56,14 @@ namespace blt::gp return allocator; } + void gp_program::save_generation(fs::writer_t& writer) + { + } + + void gp_program::load_generation(fs::reader_t& reader) + { + } + void gp_program::save_state(fs::writer_t& writer) { } diff --git a/src/sync.cpp b/src/sync.cpp index 88ab06d..62a78de 100644 --- a/src/sync.cpp +++ b/src/sync.cpp @@ -16,8 +16,79 @@ * along with this program. If not, see . */ #include +#include +#include +#include namespace blt::gp { + struct global_sync_state_t + { + std::vector syncs; + std::mutex mutex; + std::thread* thread = nullptr; + std::atomic_bool should_run = true; + std::condition_variable condition_variable; -} \ No newline at end of file + void add(sync_t* sync) + { + if (thread == nullptr) + { + thread = new std::thread([this]() + { + while (should_run) + { + std::unique_lock lock(mutex); + condition_variable.wait_for(lock, std::chrono::milliseconds(100)); + const auto current_time = system::getCurrentTimeMilliseconds(); + for (const auto& sync : syncs) + sync->trigger(current_time); + } + }); + } + std::scoped_lock lock(mutex); + syncs.push_back(sync); + } + + void remove(const sync_t* sync) + { + if (thread == nullptr) + { + BLT_WARN("Tried to remove sync from global sync state, but no thread was running"); + return; + } + std::scoped_lock lock(mutex); + const auto iter = std::find(syncs.begin(), syncs.end(), sync); + std::iter_swap(iter, syncs.end() - 1); + syncs.pop_back(); + if (syncs.empty()) + { + should_run = false; + condition_variable.notify_all(); + thread->join(); + delete thread; + thread = nullptr; + } + } + }; + + global_sync_state_t& get_state() + { + static global_sync_state_t state; + return state; + } + + sync_t::sync_t(gp_program& program): m_program(&program) + { + get_state().add(this); + } + + void sync_t::trigger(u64 current_time) + { + } + + sync_t::~sync_t() + { + get_state().remove(this); + } +} From 373fdec6d2f2f9372b42c19d16b81aef7a9d7718 Mon Sep 17 00:00:00 2001 From: Brett Date: Sat, 12 Apr 2025 16:22:37 -0400 Subject: [PATCH 06/32] silly --- CMakeLists.txt | 2 +- src/sync.cpp | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 26bcd99..dea35c2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -27,7 +27,7 @@ macro(compile_options target_name) sanitizers(${target_name}) endmacro() -project(blt-gp VERSION 0.5.1) +project(blt-gp VERSION 0.5.2) include(CTest) diff --git a/src/sync.cpp b/src/sync.cpp index 62a78de..40f8839 100644 --- a/src/sync.cpp +++ b/src/sync.cpp @@ -57,12 +57,13 @@ namespace blt::gp BLT_WARN("Tried to remove sync from global sync state, but no thread was running"); return; } - std::scoped_lock lock(mutex); + std::unique_lock lock(mutex); const auto iter = std::find(syncs.begin(), syncs.end(), sync); std::iter_swap(iter, syncs.end() - 1); syncs.pop_back(); if (syncs.empty()) { + lock.unlock(); should_run = false; condition_variable.notify_all(); thread->join(); @@ -83,8 +84,12 @@ namespace blt::gp get_state().add(this); } - void sync_t::trigger(u64 current_time) + void sync_t::trigger(const u64 current_time) { + if ((m_timer_seconds && (current_time % *m_timer_seconds == 0)) || (m_generations && (current_time % *m_generations == 0))) + { + + } } sync_t::~sync_t() From 49c80c8f53c499be2e24a966ee6bb08ab7abcb5c Mon Sep 17 00:00:00 2001 From: Brett Laptop Date: Sat, 12 Apr 2025 18:28:08 -0400 Subject: [PATCH 07/32] hi --- .idea/editor.xml | 245 ++++++++++++++++++++++++++++++++++++- CMakeLists.txt | 2 +- include/blt/gp/program.h | 2 +- include/blt/gp/sync.h | 37 +++++- include/blt/gp/threading.h | 14 +-- lib/blt | 2 +- src/sync.cpp | 15 ++- 7 files changed, 296 insertions(+), 21 deletions(-) diff --git a/.idea/editor.xml b/.idea/editor.xml index 8abdf90..1c959fd 100644 --- a/.idea/editor.xml +++ b/.idea/editor.xml @@ -1,6 +1,8 @@ + \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index dea35c2..2c72509 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -27,7 +27,7 @@ macro(compile_options target_name) sanitizers(${target_name}) endmacro() -project(blt-gp VERSION 0.5.2) +project(blt-gp VERSION 0.5.3) include(CTest) diff --git a/include/blt/gp/program.h b/include/blt/gp/program.h index bf63684..fc6bc3c 100644 --- a/include/blt/gp/program.h +++ b/include/blt/gp/program.h @@ -1058,7 +1058,7 @@ namespace blt::gp std::atomic_uint64_t next_gen_left = 0; std::atomic_bool lifetime_over = false; - blt::barrier barrier; + blt::barrier_t barrier; explicit concurrency_storage(blt::size_t threads): barrier(threads, lifetime_over) {} diff --git a/include/blt/gp/sync.h b/include/blt/gp/sync.h index 8988b6e..cbff1c3 100644 --- a/include/blt/gp/sync.h +++ b/include/blt/gp/sync.h @@ -27,9 +27,9 @@ namespace blt::gp class sync_t { public: - explicit sync_t(gp_program& program); + explicit sync_t(gp_program& program, fs::writer_t& writer); - void trigger(u64 current_time); + void trigger(u64 current_time) const; sync_t& with_timer(u64 seconds) { @@ -43,12 +43,45 @@ namespace blt::gp return *this; } + sync_t& overwrite_file_on_write() + { + m_reset_to_start_of_file = true; + return *this; + } + + sync_t& append_to_file_on_write() + { + m_reset_to_start_of_file = false; + return *this; + } + + /** + * Save the state of the whole program instead of just the generation information. + */ + sync_t& whole_program() + { + m_whole_program = true; + return *this; + } + + /** + * Only save the current generation to disk. + */ + sync_t& generation_only() + { + m_whole_program = false; + return *this; + } + ~sync_t(); private: gp_program* m_program; + fs::writer_t* m_writer; std::optional m_timer_seconds; std::optional m_generations; + bool m_reset_to_start_of_file = false; + bool m_whole_program = false; }; } diff --git a/include/blt/gp/threading.h b/include/blt/gp/threading.h index 263a896..00c91b1 100644 --- a/include/blt/gp/threading.h +++ b/include/blt/gp/threading.h @@ -97,9 +97,9 @@ namespace blt::gp task_builder_t() = default; template - static std::function make_callable(Tasks&&... tasks) + static std::function make_callable(Tasks&&... tasks) { - return [&tasks...](barrier& sync_barrier, EnumId task, size_t thread_index) + return [&tasks...](barrier_t& sync_barrier, EnumId task, size_t thread_index) { call_jmp_table(sync_barrier, task, thread_index, tasks...); }; @@ -107,7 +107,7 @@ namespace blt::gp private: template - static void execute(barrier& sync_barrier, const size_t thread_index, Task&& task) + static void execute(barrier_t& sync_barrier, const size_t thread_index, Task&& task) { // sync_barrier.wait(); if (task.requires_single_sync) @@ -121,7 +121,7 @@ namespace blt::gp } template - static bool call(barrier& sync_barrier, const EnumId current_task, const size_t thread_index, Task&& task) + static bool call(barrier_t& sync_barrier, const EnumId current_task, const size_t thread_index, Task&& task) { if (static_cast(current_task) == static_cast(task.get_task_id())) { @@ -132,7 +132,7 @@ namespace blt::gp } template - static void call_jmp_table(barrier& sync_barrier, const EnumId current_task, const size_t thread_index, Tasks&&... tasks) + static void call_jmp_table(barrier_t& sync_barrier, const EnumId current_task, const size_t thread_index, Tasks&&... tasks) { if (static_cast(current_task) >= sizeof...(tasks)) BLT_UNREACHABLE; @@ -146,7 +146,7 @@ namespace blt::gp static_assert(std::is_enum_v, "Enum ID must be of enum type!"); public: - explicit thread_manager_t(const size_t thread_count, std::function task_func, + explicit thread_manager_t(const size_t thread_count, std::function task_func, const bool will_main_block = true): barrier(thread_count), will_main_block(will_main_block) { thread_callable = [this, task_func = std::move(task_func)](const size_t thread_index) @@ -226,7 +226,7 @@ namespace blt::gp return will_main_block ? threads.size() + 1 : threads.size(); } - blt::barrier barrier; + blt::barrier_t barrier; std::atomic_bool should_run = true; bool will_main_block; std::vector tasks; diff --git a/lib/blt b/lib/blt index b6fc170..6161d9b 160000 --- a/lib/blt +++ b/lib/blt @@ -1 +1 @@ -Subproject commit b6fc1703995195af611532cdffda52c86993e4ce +Subproject commit 6161d9b79419879d9cd3374c889d124cd1e3617e diff --git a/src/sync.cpp b/src/sync.cpp index 40f8839..7519907 100644 --- a/src/sync.cpp +++ b/src/sync.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include namespace blt::gp @@ -79,16 +80,22 @@ namespace blt::gp return state; } - sync_t::sync_t(gp_program& program): m_program(&program) + sync_t::sync_t(gp_program& program, fs::writer_t& writer): m_program(&program), m_writer(&writer) { get_state().add(this); } - void sync_t::trigger(const u64 current_time) + void sync_t::trigger(const u64 current_time) const { - if ((m_timer_seconds && (current_time % *m_timer_seconds == 0)) || (m_generations && (current_time % *m_generations == 0))) + if ((m_timer_seconds && (current_time % *m_timer_seconds == 0)) || (m_generations && (m_program->get_current_generation() % *m_generations == + 0))) { - + if (m_reset_to_start_of_file) + m_writer->seek(0, fs::writer_t::seek_origin::seek_set); + if (m_whole_program) + m_program->save_state(*m_writer); + else + m_program->save_generation(*m_writer); } } From 9bc234882a6f1b67289dde4182f68e97ed40b9f5 Mon Sep 17 00:00:00 2001 From: Brett Date: Tue, 15 Apr 2025 02:21:14 -0400 Subject: [PATCH 08/32] start --- .idea/editor.xml | 239 ------------------------------------------ CMakeLists.txt | 2 +- include/blt/gp/sync.h | 2 +- src/program.cpp | 22 ++++ 4 files changed, 24 insertions(+), 241 deletions(-) diff --git a/.idea/editor.xml b/.idea/editor.xml index 1c959fd..bcfc588 100644 --- a/.idea/editor.xml +++ b/.idea/editor.xml @@ -242,244 +242,5 @@ \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 2c72509..9d17213 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -27,7 +27,7 @@ macro(compile_options target_name) sanitizers(${target_name}) endmacro() -project(blt-gp VERSION 0.5.3) +project(blt-gp VERSION 0.5.4) include(CTest) diff --git a/include/blt/gp/sync.h b/include/blt/gp/sync.h index cbff1c3..1ef3501 100644 --- a/include/blt/gp/sync.h +++ b/include/blt/gp/sync.h @@ -29,7 +29,7 @@ namespace blt::gp public: explicit sync_t(gp_program& program, fs::writer_t& writer); - void trigger(u64 current_time) const; + virtual void trigger(u64 current_time) const; sync_t& with_timer(u64 seconds) { diff --git a/src/program.cpp b/src/program.cpp index 7ac6018..5669dac 100644 --- a/src/program.cpp +++ b/src/program.cpp @@ -58,18 +58,40 @@ namespace blt::gp void gp_program::save_generation(fs::writer_t& writer) { + const auto individuals = current_pop.get_individuals().size(); + writer.write(&individuals, sizeof(individuals)); + for (const auto& individual : current_pop.get_individuals()) + { + writer.write(&individual.fitness, sizeof(individual.fitness)); + individual.tree.to_file(writer); + } } void gp_program::load_generation(fs::reader_t& reader) { + size_t individuals; + reader.read(&individuals, sizeof(individuals)); + if (current_pop.get_individuals().size() != individuals) + { + for (size_t i = current_pop.get_individuals().size(); i < individuals; i++) + current_pop.get_individuals().emplace_back(tree_t{*this}); + } + for (auto& individual : current_pop.get_individuals()) + { + reader.read(&individual.fitness, sizeof(individual.fitness)); + individual.tree.clear(*this); + individual.tree.from_file(reader); + } } void gp_program::save_state(fs::writer_t& writer) { + } void gp_program::load_state(fs::reader_t& reader) { + } void gp_program::create_threads() From 81b35f20b927c32c83898446e3ad6dcd4395d61a Mon Sep 17 00:00:00 2001 From: Brett Laptop Date: Tue, 15 Apr 2025 22:18:44 -0400 Subject: [PATCH 09/32] writing --- CMakeLists.txt | 2 +- src/program.cpp | 101 +++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 92 insertions(+), 11 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9d17213..426ddc4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -27,7 +27,7 @@ macro(compile_options target_name) sanitizers(${target_name}) endmacro() -project(blt-gp VERSION 0.5.4) +project(blt-gp VERSION 0.5.5) include(CTest) diff --git a/src/program.cpp b/src/program.cpp index 5669dac..553c9aa 100644 --- a/src/program.cpp +++ b/src/program.cpp @@ -29,20 +29,22 @@ namespace blt::gp prog_config_t::prog_config_t(): mutator(s_mutator), crossover(s_crossover), pop_initializer(s_init) { - } prog_config_t::prog_config_t(const std::reference_wrapper& popInitializer): - mutator(s_mutator), crossover(s_crossover), pop_initializer(popInitializer) - {} + mutator(s_mutator), crossover(s_crossover), pop_initializer(popInitializer) + { + } prog_config_t::prog_config_t(size_t populationSize, const std::reference_wrapper& popInitializer): - population_size(populationSize), mutator(s_mutator), crossover(s_crossover), pop_initializer(popInitializer) - {} + population_size(populationSize), mutator(s_mutator), crossover(s_crossover), pop_initializer(popInitializer) + { + } prog_config_t::prog_config_t(size_t populationSize): - population_size(populationSize), mutator(s_mutator), crossover(s_crossover), pop_initializer(s_init) - {} + population_size(populationSize), mutator(s_mutator), crossover(s_crossover), pop_initializer(s_init) + { + } random_t& gp_program::get_random() const { @@ -86,12 +88,90 @@ namespace blt::gp void gp_program::save_state(fs::writer_t& writer) { - + const size_t operator_count = storage.operators.size(); + writer.write(&operator_count, sizeof(operator_count)); + for (const auto& [i, op] : enumerate(storage.operators)) + { + writer.write(&i, sizeof(i)); + bool has_name = storage.names[i].has_value(); + writer.write(&has_name, sizeof(has_name)); + if (has_name) + { + auto size = storage.names[i]->size(); + writer.write(&size, sizeof(size)); + writer.write(storage.names[i]->data(), size); + } + writer.write(&storage.operator_metadata[i].arg_size_bytes, sizeof(storage.operator_metadata[i].arg_size_bytes)); + writer.write(&storage.operator_metadata[i].return_size_bytes, sizeof(storage.operator_metadata[i].return_size_bytes)); + writer.write(&op.argc, sizeof(op.argc)); + writer.write(&op.return_type, sizeof(op.return_type)); + const size_t argc_type_count = op.argument_types.size(); + writer.write(&argc_type_count, sizeof(argc_type_count)); + for (const auto argument : op.argument_types) + writer.write(&argument, sizeof(argument)); + } + save_generation(writer); } void gp_program::load_state(fs::reader_t& reader) { + size_t operator_count; + reader.read(&operator_count, sizeof(operator_count)); + if (operator_count != storage.operators.size()) + throw std::runtime_error( + "Invalid number of operators. Expected " + std::to_string(storage.operators.size()) + " found " + std::to_string(operator_count)); + for (size_t i = 0; i < operator_count; i++) + { + size_t expected_i; + reader.read(&expected_i, sizeof(expected_i)); + if (expected_i != i) + throw std::runtime_error("Loaded invalid operator ID. Expected " + std::to_string(i) + " found " + std::to_string(expected_i)); + bool has_name; + reader.read(&has_name, sizeof(has_name)); + if (has_name) + { + size_t size; + reader.read(&size, sizeof(size)); + std::string name; + name.resize(size); + reader.read(name.data(), size); + if (!storage.names[i].has_value()) + throw std::runtime_error("Expected operator ID " + std::to_string(i) + " to have name " + name); + if (name != *storage.names[i]) + throw std::runtime_error( + "Operator ID " + std::to_string(i) + " expected to be named " + name + " found " + std::string(*storage.names[i])); + auto& op = storage.operators[i]; + auto& op_meta = storage.operator_metadata[i]; + decltype(std::declval().arg_size_bytes) arg_size_bytes; + decltype(std::declval().return_size_bytes) return_size_bytes; + reader.read(&arg_size_bytes, sizeof(arg_size_bytes)); + reader.read(&return_size_bytes, sizeof(return_size_bytes)); + + if (op_meta.arg_size_bytes != arg_size_bytes) + throw std::runtime_error( + "Operator ID " + std::to_string(i) + " expected operator to take " + std::to_string(op_meta.arg_size_bytes) + " but got " + + std::to_string(arg_size_bytes)); + + if (op_meta.return_size_bytes != return_size_bytes) + throw std::runtime_error( + "Operator ID " + std::to_string(i) + " expected operator to return " + std::to_string(op_meta.return_size_bytes) + " but got " + + std::to_string(return_size_bytes)); + + argc_t argc; + reader.read(&argc, sizeof(argc)); + type_id return_type; + reader.read(&return_type, sizeof(return_type)); + size_t arg_type_count; + reader.read(&arg_type_count, sizeof(arg_type_count)); + for (size_t j = 0; j < arg_type_count; j++) + { + type_id type; + reader.read(&type, sizeof(type)); + } + } + } + load_generation(reader); } void gp_program::create_threads() @@ -105,7 +185,8 @@ namespace blt::gp // main thread is thread0 for (blt::size_t i = 1; i < config.threads; i++) { - thread_helper.threads.emplace_back(new std::thread([i, this]() { + thread_helper.threads.emplace_back(new std::thread([i, this]() + { #ifdef BLT_TRACK_ALLOCATIONS tracker.reserve(); tracker.await_thread_loading_complete(config.threads); @@ -133,4 +214,4 @@ namespace blt::gp tracker.await_thread_loading_complete(config.threads); #endif } -} \ No newline at end of file +} From f89628615ad9a2ac3d3096e273b11ed6b80e0990 Mon Sep 17 00:00:00 2001 From: Brett Laptop Date: Wed, 16 Apr 2025 23:27:34 -0400 Subject: [PATCH 10/32] uwu --- CMakeLists.txt | 2 +- src/program.cpp | 63 ++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 63 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 426ddc4..f31d958 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -27,7 +27,7 @@ macro(compile_options target_name) sanitizers(${target_name}) endmacro() -project(blt-gp VERSION 0.5.5) +project(blt-gp VERSION 0.5.6) include(CTest) diff --git a/src/program.cpp b/src/program.cpp index 553c9aa..f6aea3b 100644 --- a/src/program.cpp +++ b/src/program.cpp @@ -86,6 +86,35 @@ namespace blt::gp } } + void write_stat(fs::writer_t& writer, const population_stats& stat) + { + const auto overall_fitness = stat.overall_fitness.load(); + const auto average_fitness = stat.average_fitness.load(); + const auto best_fitness = stat.best_fitness.load(); + const auto worst_fitness = stat.worst_fitness.load(); + writer.write(&overall_fitness, sizeof(overall_fitness)); + writer.write(&average_fitness, sizeof(average_fitness)); + writer.write(&best_fitness, sizeof(best_fitness)); + writer.write(&worst_fitness, sizeof(worst_fitness)); + const size_t fitness_count = stat.normalized_fitness.size(); + writer.write(&fitness_count, sizeof(fitness_count)); + for (const auto& fitness : stat.normalized_fitness) + writer.write(&fitness, sizeof(fitness)); + } + + void load_stat(fs::reader_t& reader, population_stats& stat) + { + reader.read(&stat.overall_fitness, sizeof(stat.overall_fitness)); + reader.read(&stat.average_fitness, sizeof(stat.average_fitness)); + reader.read(&stat.best_fitness, sizeof(stat.best_fitness)); + reader.read(&stat.worst_fitness, sizeof(stat.worst_fitness)); + size_t fitness_count; + reader.read(&fitness_count, sizeof(fitness_count)); + stat.normalized_fitness.resize(fitness_count); + for (auto& fitness : stat.normalized_fitness) + reader.read(&fitness, sizeof(fitness)); + } + void gp_program::save_state(fs::writer_t& writer) { const size_t operator_count = storage.operators.size(); @@ -110,6 +139,11 @@ namespace blt::gp for (const auto argument : op.argument_types) writer.write(&argument, sizeof(argument)); } + const size_t history_count = statistic_history.size(); + writer.write(&history_count, sizeof(history_count)); + for (const auto& stat : statistic_history) + write_stat(writer, stat); + write_stat(writer, current_stats); save_generation(writer); } @@ -155,22 +189,49 @@ namespace blt::gp if (op_meta.return_size_bytes != return_size_bytes) throw std::runtime_error( - "Operator ID " + std::to_string(i) + " expected operator to return " + std::to_string(op_meta.return_size_bytes) + " but got " + + "Operator ID " + std::to_string(i) + " expected operator to return " + std::to_string(op_meta.return_size_bytes) + " but got " + + std::to_string(return_size_bytes)); argc_t argc; reader.read(&argc, sizeof(argc)); + if (argc.argc != op.argc.argc) + throw std::runtime_error( + "Operator ID " + std::to_string(i) + " expected " + std::to_string(op.argc.argc) + " arguments but got " + std::to_string( + argc.argc)); + if (argc.argc_context != op.argc.argc_context) + throw std::runtime_error( + "Operator ID " + std::to_string(i) + " expected " + std::to_string(op.argc.argc_context) + " arguments but got " + + std::to_string(argc.argc_context)); type_id return_type; reader.read(&return_type, sizeof(return_type)); + if (return_type != op.return_type) + throw std::runtime_error( + "Operator ID " + std::to_string(i) + " expected return type " + std::to_string(op.return_type) + " but got " + std::to_string( + return_type)); size_t arg_type_count; + if (arg_type_count != op.argument_types.size()) + throw std::runtime_error( + "Operator ID " + std::to_string(i) + " expected " + std::to_string(op.argument_types.size()) + " arguments but got " + + std::to_string(arg_type_count)); reader.read(&arg_type_count, sizeof(arg_type_count)); for (size_t j = 0; j < arg_type_count; j++) { type_id type; reader.read(&type, sizeof(type)); + if (type != op.argument_types[j]) + throw std::runtime_error( + "Operator ID " + std::to_string(i) + " expected argument " + std::to_string(j) + " to be of type " + std::to_string( + op.argument_types[j]) + " but got " + std::to_string(type)); } } } + size_t history_count; + reader.read(&history_count, sizeof(history_count)); + statistic_history.resize(history_count); + for (size_t i = 0; i < history_count; i++) + load_stat(reader, statistic_history[i]); + load_stat(reader, current_stats); load_generation(reader); } From 0d299affdbed8d773489f252cb13a1abbe9ffb44 Mon Sep 17 00:00:00 2001 From: Brett Date: Thu, 17 Apr 2025 02:15:16 -0400 Subject: [PATCH 11/32] why is it broken on nix? --- .idea/editor.xml | 455 ++++++++++++++++++----------------- CMakeLists.txt | 3 +- lib/blt | 2 +- tests/serialization_test.cpp | 104 ++++++++ 4 files changed, 336 insertions(+), 228 deletions(-) create mode 100644 tests/serialization_test.cpp diff --git a/.idea/editor.xml b/.idea/editor.xml index bcfc588..329a4e3 100644 --- a/.idea/editor.xml +++ b/.idea/editor.xml @@ -1,246 +1,249 @@ - \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index f31d958..8e090d4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -27,7 +27,7 @@ macro(compile_options target_name) sanitizers(${target_name}) endmacro() -project(blt-gp VERSION 0.5.6) +project(blt-gp VERSION 0.5.7) include(CTest) @@ -123,5 +123,6 @@ if (${BUILD_GP_TESTS}) blt_add_project(blt-symbolic-regression tests/symbolic_regression_test.cpp test) blt_add_project(blt-drop tests/drop_test.cpp test) blt_add_project(blt-drop-2-type tests/2_type_drop_test.cpp test) + blt_add_project(blt-serialization tests/serialization_test.cpp test) endif () \ No newline at end of file diff --git a/lib/blt b/lib/blt index 6161d9b..6cdfab3 160000 --- a/lib/blt +++ b/lib/blt @@ -1 +1 @@ -Subproject commit 6161d9b79419879d9cd3374c889d124cd1e3617e +Subproject commit 6cdfab39cfc1e10fe8fe7a34863963d01620bdb3 diff --git a/tests/serialization_test.cpp b/tests/serialization_test.cpp new file mode 100644 index 0000000..9be2246 --- /dev/null +++ b/tests/serialization_test.cpp @@ -0,0 +1,104 @@ +/* + * + * Copyright (C) 2025 Brett Terpstra + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#include "../examples/symbolic_regression.h" +#include +#include + +using namespace blt::gp; + +struct context +{ + float x, y; +}; + +prog_config_t config = prog_config_t() + .set_initial_min_tree_size(2) + .set_initial_max_tree_size(6) + .set_elite_count(2) + .set_crossover_chance(0.8) + .set_mutation_chance(0.1) + .set_reproduction_chance(0.1) + .set_max_generations(50) + .set_pop_size(500) + .set_thread_count(1); + + +example::symbolic_regression_t regression{691ul, config}; + +operation_t addf{[](const float a, const float b) { return a + b; }, "addf"}; +operation_t subf([](const float a, const float b) { return a - b; }, "subf"); +operation_t mulf([](const float a, const float b) { return a * b; }, "mulf"); +operation_t pro_divf([](const float a, const float b) { return b == 0.0f ? 0.0f : a / b; }, "divf"); +operation_t op_sinf([](const float a) { return std::sin(a); }, "sinf"); +operation_t op_cosf([](const float a) { return std::cos(a); }, "cosf"); +operation_t op_expf([](const float a) { return std::exp(a); }, "expf"); +operation_t op_logf([](const float a) { return a <= 0.0f ? 0.0f : std::log(a); }, "logf"); + +auto litf = operation_t([]() +{ + return regression.get_program().get_random().get_float(-1.0f, 1.0f); +}, "litf").set_ephemeral(); + +operation_t op_xf([](const context& context) +{ + return context.x; +}, "xf"); + +bool fitness_function(const tree_t& current_tree, fitness_t& fitness, size_t) +{ + constexpr static double value_cutoff = 1.e15; + for (auto& fitness_case : regression.get_training_cases()) + { + BLT_GP_UPDATE_CONTEXT(fitness_case); + auto val = current_tree.get_evaluation_ref(fitness_case); + const auto diff = std::abs(fitness_case.y - val.get().value()); + if (diff < value_cutoff) + { + fitness.raw_fitness += diff; + if (diff <= 0.01) + fitness.hits++; + } + else + fitness.raw_fitness += value_cutoff; + } + fitness.standardized_fitness = fitness.raw_fitness; + fitness.adjusted_fitness = (1.0 / (1.0 + fitness.standardized_fitness)); + return static_cast(fitness.hits) == regression.get_training_cases().size(); +} + +int main() +{ + operator_builder builder{}; + builder.build(addf, subf, mulf, pro_divf, op_sinf, op_cosf, op_expf, op_logf, litf, op_xf); + regression.get_program().set_operations(builder.grab()); + + auto& program = regression.get_program(); + static auto sel = select_tournament_t{}; + program.generate_initial_population(program.get_typesystem().get_type().id()); + program.setup_generational_evaluation(fitness_function, sel, sel, sel); + while (!program.should_terminate()) + { + BLT_TRACE("---------------\\{Begin Generation {}}---------------", program.get_current_generation()); + BLT_TRACE("Creating next generation"); + program.create_next_generation(); + BLT_TRACE("Move to next generation"); + program.next_generation(); + BLT_TRACE("Evaluate Fitness"); + program.evaluate_fitness(); + } +} From 29d8efe18f060c6de644ec1991b68a43707a3d72 Mon Sep 17 00:00:00 2001 From: Brett Date: Thu, 17 Apr 2025 02:20:33 -0400 Subject: [PATCH 12/32] fix --- CMakeLists.txt | 2 +- lib/blt | 2 +- tests/serialization_test.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8e090d4..d63341f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -27,7 +27,7 @@ macro(compile_options target_name) sanitizers(${target_name}) endmacro() -project(blt-gp VERSION 0.5.7) +project(blt-gp VERSION 0.5.8) include(CTest) diff --git a/lib/blt b/lib/blt index 6cdfab3..90cf177 160000 --- a/lib/blt +++ b/lib/blt @@ -1 +1 @@ -Subproject commit 6cdfab39cfc1e10fe8fe7a34863963d01620bdb3 +Subproject commit 90cf177c57f542ecdd132eaa6af0f1945ce8fe96 diff --git a/tests/serialization_test.cpp b/tests/serialization_test.cpp index 9be2246..3512c56 100644 --- a/tests/serialization_test.cpp +++ b/tests/serialization_test.cpp @@ -66,7 +66,7 @@ bool fitness_function(const tree_t& current_tree, fitness_t& fitness, size_t) { BLT_GP_UPDATE_CONTEXT(fitness_case); auto val = current_tree.get_evaluation_ref(fitness_case); - const auto diff = std::abs(fitness_case.y - val.get().value()); + const auto diff = std::abs(fitness_case.y - val.get()); if (diff < value_cutoff) { fitness.raw_fitness += diff; From dcefa93d29c60d148c98d64970e6a0c726bc4bbd Mon Sep 17 00:00:00 2001 From: Brett Date: Thu, 17 Apr 2025 17:45:32 -0400 Subject: [PATCH 13/32] serial broken --- .idea/editor.xml | 238 ---- CMakeLists.txt | 2 +- examples/rice_classification.h | 4 +- examples/src/rice_classification.cpp | 4 +- include/blt/gp/program.h | 1852 +++++++++++++------------- include/blt/gp/tree.h | 16 + src/tree.cpp | 35 +- tests/serialization_test.cpp | 37 +- 8 files changed, 1028 insertions(+), 1160 deletions(-) diff --git a/.idea/editor.xml b/.idea/editor.xml index 329a4e3..6df7d16 100644 --- a/.idea/editor.xml +++ b/.idea/editor.xml @@ -1,248 +1,10 @@ - diff --git a/CMakeLists.txt b/CMakeLists.txt index d63341f..3a9ff7b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -27,7 +27,7 @@ macro(compile_options target_name) sanitizers(${target_name}) endmacro() -project(blt-gp VERSION 0.5.8) +project(blt-gp VERSION 0.5.9) include(CTest) diff --git a/examples/rice_classification.h b/examples/rice_classification.h index 2b03156..b779e2f 100644 --- a/examples/rice_classification.h +++ b/examples/rice_classification.h @@ -116,8 +116,8 @@ namespace blt::gp::example mutation_sel = &sel; if (reproduction_sel == nullptr) reproduction_sel = &sel; - program.generate_population(program.get_typesystem().get_type().id(), fitness_function_ref, *crossover_sel, *mutation_sel, - *reproduction_sel); + program.generate_initial_population(program.get_typesystem().get_type().id()); + program.setup_generational_evaluation(fitness_function_ref, *crossover_sel, *mutation_sel, *reproduction_sel); } void print_best(const size_t amount = 3) diff --git a/examples/src/rice_classification.cpp b/examples/src/rice_classification.cpp index 8f8b95b..b31325d 100644 --- a/examples/src/rice_classification.cpp +++ b/examples/src/rice_classification.cpp @@ -21,7 +21,7 @@ #include #include #include -#include +#include #include #include #include @@ -191,7 +191,7 @@ void blt::gp::example::rice_classification_t::load_rice_data(const std::string_v training_cases.insert(training_cases.end(), c.begin(), c.end()); training_cases.insert(training_cases.end(), o.begin(), o.end()); std::shuffle(training_cases.begin(), training_cases.end(), program.get_random()); - BLT_INFO("Created testing set of size %ld, training set is of size %ld", testing_cases.size(), training_cases.size()); + BLT_INFO("Created testing set of size {}, training set is of size {}", testing_cases.size(), training_cases.size()); } blt::gp::confusion_matrix_t blt::gp::example::rice_classification_t::test_individual(const individual_t& individual) const diff --git a/include/blt/gp/program.h b/include/blt/gp/program.h index fc6bc3c..25132ca 100644 --- a/include/blt/gp/program.h +++ b/include/blt/gp/program.h @@ -56,309 +56,316 @@ namespace blt::gp { - struct argc_t - { - blt::u32 argc = 0; - blt::u32 argc_context = 0; + struct argc_t + { + blt::u32 argc = 0; + blt::u32 argc_context = 0; - [[nodiscard]] bool is_terminal() const - { - return argc == 0; - } - }; + [[nodiscard]] bool is_terminal() const + { + return argc == 0; + } + }; - struct operator_info_t - { - // types of the arguments - tracked_vector argument_types; - // return type of this operator - type_id return_type; - // number of arguments for this operator - argc_t argc; - // per operator function callable (slow) - detail::operator_func_t func; - }; + struct operator_info_t + { + // types of the arguments + tracked_vector argument_types; + // return type of this operator + type_id return_type; + // number of arguments for this operator + argc_t argc; + // per operator function callable (slow) + detail::operator_func_t func; + }; - struct operator_metadata_t - { - blt::size_t arg_size_bytes = 0; - blt::size_t return_size_bytes = 0; - argc_t argc{}; - }; + struct operator_metadata_t + { + blt::size_t arg_size_bytes = 0; + blt::size_t return_size_bytes = 0; + argc_t argc{}; + }; - struct program_operator_storage_t - { - // indexed from return TYPE ID, returns index of operator - expanding_buffer> terminals; - expanding_buffer> non_terminals; - expanding_buffer>> operators_ordered_terminals; - // indexed from OPERATOR ID (operator number) to a bitfield of flags - hashmap_t operator_flags; + struct program_operator_storage_t + { + // indexed from return TYPE ID, returns index of operator + expanding_buffer> terminals; + expanding_buffer> non_terminals; + expanding_buffer>> operators_ordered_terminals; + // indexed from OPERATOR ID (operator number) to a bitfield of flags + hashmap_t operator_flags; - tracked_vector operators; - tracked_vector operator_metadata; - tracked_vector print_funcs; - tracked_vector destroy_funcs; - tracked_vector> names; + tracked_vector operators; + tracked_vector operator_metadata; + tracked_vector print_funcs; + tracked_vector destroy_funcs; + tracked_vector> names; - detail::eval_func_t eval_func; + detail::eval_func_t eval_func; - type_provider system; - }; + type_provider system; + }; - template - class operator_builder - { - friend class gp_program; + template + class operator_builder + { + friend class gp_program; - friend class blt::gp::detail::operator_storage_test; + friend class blt::gp::detail::operator_storage_test; - public: - explicit operator_builder() = default; + public: + explicit operator_builder() = default; - template - program_operator_storage_t& build(Operators&... operators) - { - blt::size_t largest_args = 0; - blt::size_t largest_returns = 0; - blt::u32 largest_argc = 0; - operator_metadata_t meta; - ((meta = add_operator(operators), largest_argc = std::max(meta.argc.argc, largest_argc), largest_args = - std::max(meta.arg_size_bytes, largest_args), largest_returns = std::max(meta.return_size_bytes, largest_returns)), ...); + template + program_operator_storage_t& build(Operators&... operators) + { + blt::size_t largest_args = 0; + blt::size_t largest_returns = 0; + blt::u32 largest_argc = 0; + operator_metadata_t meta; + ((meta = add_operator(operators), largest_argc = std::max(meta.argc.argc, largest_argc), largest_args = + std::max(meta.arg_size_bytes, largest_args), largest_returns = std::max(meta.return_size_bytes, largest_returns)), ...); - // largest = largest * largest_argc; - size_t largest = largest_args * largest_argc * largest_returns * largest_argc; + // largest = largest * largest_argc; + size_t largest = largest_args * largest_argc * largest_returns * largest_argc; - storage.eval_func = tree_t::make_execution_lambda(largest, operators...); + storage.eval_func = tree_t::make_execution_lambda(largest, operators...); - blt::hashset_t has_terminals; + blt::hashset_t has_terminals; - for (const auto& [index, value] : blt::enumerate(storage.terminals)) - { - if (!value.empty()) - has_terminals.insert(index); - } + for (const auto& [index, value] : blt::enumerate(storage.terminals)) + { + if (!value.empty()) + has_terminals.insert(index); + } - for (const auto& [index, value] : blt::enumerate(storage.non_terminals)) - { - if (value.empty()) - continue; - auto return_type = index; - tracked_vector> ordered_terminals; - for (const auto& op : value) - { - // count number of terminals - blt::size_t terminals = 0; - for (const auto& type : storage.operators[op].argument_types) - { - if (has_terminals.contains(type)) - terminals++; - } - ordered_terminals.emplace_back(op, terminals); - } - bool found_terminal_inputs = false; - bool matches_argc = false; - for (const auto& terms : ordered_terminals) - { - if (terms.second == storage.operators[terms.first].argc.argc) - matches_argc = true; - if (terms.second != 0) - found_terminal_inputs = true; - if (matches_argc && found_terminal_inputs) - break; - } - if (!found_terminal_inputs) - BLT_ABORT(("Failed to find function with terminal arguments for return type " + std::to_string(return_type)).c_str()); - if (!matches_argc) - { - BLT_ABORT(("Failed to find a function which purely translates types " - "(that is all input types are terminals) for return type " + std::to_string(return_type)).c_str()); - } + for (const auto& [index, value] : blt::enumerate(storage.non_terminals)) + { + if (value.empty()) + continue; + auto return_type = index; + tracked_vector> ordered_terminals; + for (const auto& op : value) + { + // count number of terminals + blt::size_t terminals = 0; + for (const auto& type : storage.operators[op].argument_types) + { + if (has_terminals.contains(type)) + terminals++; + } + ordered_terminals.emplace_back(op, terminals); + } + bool found_terminal_inputs = false; + bool matches_argc = false; + for (const auto& terms : ordered_terminals) + { + if (terms.second == storage.operators[terms.first].argc.argc) + matches_argc = true; + if (terms.second != 0) + found_terminal_inputs = true; + if (matches_argc && found_terminal_inputs) + break; + } + if (!found_terminal_inputs) + BLT_ABORT(("Failed to find function with terminal arguments for return type " + std::to_string(return_type)).c_str()); + if (!matches_argc) + { + BLT_ABORT(("Failed to find a function which purely translates types " + "(that is all input types are terminals) for return type " + std::to_string(return_type)).c_str()); + } - std::sort(ordered_terminals.begin(), ordered_terminals.end(), [](const auto& a, const auto& b) { - return a.second > b.second; - }); + std::sort(ordered_terminals.begin(), ordered_terminals.end(), [](const auto& a, const auto& b) + { + return a.second > b.second; + }); - auto first_size = *ordered_terminals.begin(); - auto iter = ordered_terminals.begin(); - while (++iter != ordered_terminals.end() && iter->second == first_size.second) - {} + auto first_size = *ordered_terminals.begin(); + auto iter = ordered_terminals.begin(); + while (++iter != ordered_terminals.end() && iter->second == first_size.second) + { + } - ordered_terminals.erase(iter, ordered_terminals.end()); + ordered_terminals.erase(iter, ordered_terminals.end()); - storage.operators_ordered_terminals[return_type] = ordered_terminals; - } + storage.operators_ordered_terminals[return_type] = ordered_terminals; + } - return storage; - } + return storage; + } - program_operator_storage_t&& grab() - { - return std::move(storage); - } + program_operator_storage_t&& grab() + { + return std::move(storage); + } - private: - template - auto add_operator(operation_t& op) - { - // check for types we can register - (storage.system.register_type(), ...); - storage.system.register_type(); + private: + template + auto add_operator(operation_t& op) + { + // check for types we can register + (storage.system.register_type(), ...); + storage.system.register_type(); - auto return_type_id = storage.system.get_type().id(); - auto operator_id = blt::gp::operator_id(storage.operators.size()); - op.id = operator_id; + auto return_type_id = storage.system.get_type().id(); + auto operator_id = blt::gp::operator_id(storage.operators.size()); + op.id = operator_id; - operator_info_t info; + operator_info_t info; - if constexpr (sizeof...(Args) > 0) - { - (add_non_context_argument>(info.argument_types), ...); - } + if constexpr (sizeof...(Args) > 0) + { + (add_non_context_argument>(info.argument_types), ...); + } - info.argc.argc_context = info.argc.argc = sizeof...(Args); - info.return_type = return_type_id; - info.func = op.template make_callable(); + info.argc.argc_context = info.argc.argc = sizeof...(Args); + info.return_type = return_type_id; + info.func = op.template make_callable(); - ((std::is_same_v, Context> ? info.argc.argc -= 1 : 0), ...); + ((std::is_same_v, Context> ? info.argc.argc -= 1 : 0), ...); - auto& operator_list = info.argc.argc == 0 ? storage.terminals : storage.non_terminals; - operator_list[return_type_id].push_back(operator_id); + auto& operator_list = info.argc.argc == 0 ? storage.terminals : storage.non_terminals; + operator_list[return_type_id].push_back(operator_id); - BLT_ASSERT(info.argc.argc_context - info.argc.argc <= 1 && "Cannot pass multiple context as arguments!"); + BLT_ASSERT(info.argc.argc_context - info.argc.argc <= 1 && "Cannot pass multiple context as arguments!"); - storage.operators.push_back(info); + storage.operators.push_back(info); - operator_metadata_t meta; - if constexpr (sizeof...(Args) != 0) - { - meta.arg_size_bytes = (stack_allocator::aligned_size() + ...); - } - meta.return_size_bytes = stack_allocator::aligned_size(); - meta.argc = info.argc; + operator_metadata_t meta; + if constexpr (sizeof...(Args) != 0) + { + meta.arg_size_bytes = (stack_allocator::aligned_size() + ...); + } + meta.return_size_bytes = stack_allocator::aligned_size(); + meta.argc = info.argc; - storage.operator_metadata.push_back(meta); - storage.print_funcs.push_back([&op](std::ostream& out, stack_allocator& stack) { - if constexpr (blt::meta::is_streamable_v) - { - out << stack.from(0); - (void) (op); // remove warning - } else - { - out << "[Printing Value on '" << (op.get_name() ? *op.get_name() : "") << "' Not Supported!]"; - } - }); - storage.destroy_funcs.push_back([](const detail::destroy_t type, u8* data) { - switch (type) - { - case detail::destroy_t::PTR: - case detail::destroy_t::RETURN: - if constexpr (detail::has_func_drop_v>) - { - reinterpret_cast*>(data)->drop(); - } - break; - } - }); - storage.names.push_back(op.get_name()); - storage.operator_flags.emplace(operator_id, operator_special_flags{op.is_ephemeral(), op.return_has_ephemeral_drop()}); - return meta; - } + storage.operator_metadata.push_back(meta); + storage.print_funcs.push_back([&op](std::ostream& out, stack_allocator& stack) + { + if constexpr (blt::meta::is_streamable_v) + { + out << stack.from(0); + (void)(op); // remove warning + } + else + { + out << "[Printing Value on '" << (op.get_name() ? *op.get_name() : "") << "' Not Supported!]"; + } + }); + storage.destroy_funcs.push_back([](const detail::destroy_t type, u8* data) + { + switch (type) + { + case detail::destroy_t::PTR: + case detail::destroy_t::RETURN: + if constexpr (detail::has_func_drop_v>) + { + reinterpret_cast*>(data)->drop(); + } + break; + } + }); + storage.names.push_back(op.get_name()); + storage.operator_flags.emplace(operator_id, operator_special_flags{op.is_ephemeral(), op.return_has_ephemeral_drop()}); + return meta; + } - template - void add_non_context_argument(decltype(operator_info_t::argument_types)& types) - { - if constexpr (!std::is_same_v>) - { - types.push_back(storage.system.get_type().id()); - } - } + template + void add_non_context_argument(decltype(operator_info_t::argument_types)& types) + { + if constexpr (!std::is_same_v>) + { + types.push_back(storage.system.get_type().id()); + } + } - private: - program_operator_storage_t storage; - }; + private: + program_operator_storage_t storage; + }; - class gp_program - { - public: - /** - * Note about context size: This is required as context is passed to every operator in the GP tree, this context will be provided by your - * call to one of the evaluator functions. This was the nicest way to provide this as C++ lacks reflection - * - * @param seed - */ - explicit gp_program(blt::u64 seed): seed_func([seed] { - return seed; - }) - { - create_threads(); - set_config(config); - } + class gp_program + { + public: + /** + * Note about context size: This is required as context is passed to every operator in the GP tree, this context will be provided by your + * call to one of the evaluator functions. This was the nicest way to provide this as C++ lacks reflection + * + * @param seed + */ + explicit gp_program(blt::u64 seed): seed_func([seed] + { + return seed; + }) + { + create_threads(); + set_config(config); + } - explicit gp_program(blt::u64 seed, const prog_config_t& config): seed_func([seed] { - return seed; - }) - { - create_threads(); - set_config(config); - } + explicit gp_program(blt::u64 seed, const prog_config_t& config): seed_func([seed] + { + return seed; + }) + { + create_threads(); + set_config(config); + } - /** - * - * @param seed_func Function which provides a new random seed every time it is called. - * This will be used by each thread to initialize a new random number generator - */ - explicit gp_program(std::function seed_func): seed_func(std::move(seed_func)) - { - create_threads(); - set_config(config); - } + /** + * + * @param seed_func Function which provides a new random seed every time it is called. + * This will be used by each thread to initialize a new random number generator + */ + explicit gp_program(std::function seed_func): seed_func(std::move(seed_func)) + { + create_threads(); + set_config(config); + } - explicit gp_program(std::function seed_func, const prog_config_t& config): seed_func(std::move(seed_func)) - { - create_threads(); - set_config(config); - } + explicit gp_program(std::function seed_func, const prog_config_t& config): seed_func(std::move(seed_func)) + { + create_threads(); + set_config(config); + } - ~gp_program() - { - thread_helper.lifetime_over = true; - thread_helper.barrier.notify_all(); - thread_helper.thread_function_condition.notify_all(); - for (auto& thread : thread_helper.threads) - { - if (thread->joinable()) - thread->join(); - } - } + ~gp_program() + { + thread_helper.lifetime_over = true; + thread_helper.barrier.notify_all(); + thread_helper.thread_function_condition.notify_all(); + for (auto& thread : thread_helper.threads) + { + if (thread->joinable()) + thread->join(); + } + } - void create_next_generation() - { - #ifdef BLT_TRACK_ALLOCATIONS + void create_next_generation() + { +#ifdef BLT_TRACK_ALLOCATIONS auto gen_alloc = blt::gp::tracker.start_measurement(); - #endif - // should already be empty - thread_helper.next_gen_left.store(selection_probabilities.replacement_amount.value_or(config.population_size), std::memory_order_release); - (*thread_execution_service)(0); - #ifdef BLT_TRACK_ALLOCATIONS +#endif + // should already be empty + thread_helper.next_gen_left.store(selection_probabilities.replacement_amount.value_or(config.population_size), std::memory_order_release); + (*thread_execution_service)(0); +#ifdef BLT_TRACK_ALLOCATIONS blt::gp::tracker.stop_measurement(gen_alloc); gen_alloc.pretty_print("Generation"); - #endif - } +#endif + } - void next_generation() - { - std::swap(current_pop, next_pop); - ++current_generation; - } + void next_generation() + { + std::swap(current_pop, next_pop); + ++current_generation; + } - void evaluate_fitness() - { - #ifdef BLT_TRACK_ALLOCATIONS + void evaluate_fitness() + { +#ifdef BLT_TRACK_ALLOCATIONS auto fitness_alloc = blt::gp::tracker.start_measurement(); - #endif - evaluate_fitness_internal(); - #ifdef BLT_TRACK_ALLOCATIONS +#endif + evaluate_fitness_internal(); +#ifdef BLT_TRACK_ALLOCATIONS blt::gp::tracker.stop_measurement(fitness_alloc); fitness_alloc.pretty_print("Fitness"); evaluation_calls.call(); @@ -367,566 +374,597 @@ namespace blt::gp { evaluation_allocations.call(fitness_alloc.getAllocatedByteDifference()); } - #endif - } - - void reset_program(type_id root_type, bool eval_fitness_now = true) - { - current_generation = 0; - current_pop = config.pop_initializer.get().generate({ - *this, root_type, config.population_size, config.initial_min_tree_size, config.initial_max_tree_size - }); - next_pop = population_t(current_pop); - BLT_ASSERT_MSG(current_pop.get_individuals().size() == config.population_size, - ("cur pop size: " + std::to_string(current_pop.get_individuals().size())).c_str()); - BLT_ASSERT_MSG(next_pop.get_individuals().size() == config.population_size, - ("next pop size: " + std::to_string(next_pop.get_individuals().size())).c_str()); - if (eval_fitness_now) - evaluate_fitness_internal(); - } - - void kill() - { - thread_helper.lifetime_over = true; - } - - void generate_initial_population(const type_id root_type) - { - current_pop = config.pop_initializer.get().generate({ - *this, root_type, config.population_size, config.initial_min_tree_size, config.initial_max_tree_size - }); - next_pop = population_t(current_pop); - BLT_ASSERT_MSG(current_pop.get_individuals().size() == config.population_size, - ("cur pop size: " + std::to_string(current_pop.get_individuals().size())).c_str()); - BLT_ASSERT_MSG(next_pop.get_individuals().size() == config.population_size, - ("next pop size: " + std::to_string(next_pop.get_individuals().size())).c_str()); - } - - /** - * takes in a reference to a function for the fitness evaluation function (must return a value convertable to double) - * The lambda must accept a tree for evaluation, and an index (current tree) - * - * tree_t& current_tree, blt::size_t index_of_tree - * - * Container must be concurrently accessible from multiple threads using operator[] - * - * NOTE: the larger the adjusted fitness, the better. - */ - template - void setup_generational_evaluation(FitnessFunc& fitness_function, Crossover& crossover_selection, Mutation& mutation_selection, - Reproduction& reproduction_selection, bool eval_fitness_now = true) - { - if (config.threads == 1) - { - BLT_INFO("Starting generational with single thread variant!"); - thread_execution_service = std::unique_ptr>(new std::function( - [this, &fitness_function, &crossover_selection, &mutation_selection, &reproduction_selection](size_t) { - single_threaded_fitness_eval()(fitness_function); - - if (thread_helper.next_gen_left > 0) - { - current_stats.normalized_fitness.clear(); - double sum_of_prob = 0; - for (const auto& ind : current_pop) - { - const auto prob = (ind.fitness.adjusted_fitness / current_stats.overall_fitness); - current_stats.normalized_fitness.push_back(sum_of_prob + prob); - sum_of_prob += prob; - } - - const auto args = get_selector_args(); - - crossover_selection.pre_process(*this, current_pop); - mutation_selection.pre_process(*this, current_pop); - reproduction_selection.pre_process(*this, current_pop); - - size_t start = detail::perform_elitism(args, next_pop); - - while (start < config.population_size) - { - tree_t& c1 = next_pop.get_individuals()[start].tree; - tree_t* c2 = nullptr; - if (start + 1 < config.population_size) - c2 = &next_pop.get_individuals()[start + 1].tree; - start += perform_selection(crossover_selection, mutation_selection, reproduction_selection, c1, c2); - } - - thread_helper.next_gen_left = 0; - } - })); - } else - { - BLT_INFO("Starting generational thread execution service!"); - std::scoped_lock lock(thread_helper.thread_function_control); - thread_execution_service = std::unique_ptr>(new std::function( - [this, &fitness_function, &crossover_selection, &mutation_selection, &reproduction_selection](const size_t id) { - thread_helper.barrier.wait(); - - multi_threaded_fitness_eval()(fitness_function, id); - - if (thread_helper.next_gen_left > 0) - { - thread_helper.barrier.wait(); - auto args = get_selector_args(); - if (id == 0) - { - current_stats.normalized_fitness.clear(); - double sum_of_prob = 0; - for (const auto& ind : current_pop) - { - const auto prob = (ind.fitness.adjusted_fitness / current_stats.overall_fitness); - current_stats.normalized_fitness.push_back(sum_of_prob + prob); - sum_of_prob += prob; - } - - crossover_selection.pre_process(*this, current_pop); - if (&crossover_selection != &mutation_selection) - mutation_selection.pre_process(*this, current_pop); - if (&crossover_selection != &reproduction_selection) - reproduction_selection.pre_process(*this, current_pop); - const auto elite_amount = detail::perform_elitism(args, next_pop); - thread_helper.next_gen_left -= elite_amount; - } - thread_helper.barrier.wait(); - - while (thread_helper.next_gen_left > 0) - { - size_t size = 0; - size_t begin = 0; - size_t end = thread_helper.next_gen_left.load(std::memory_order_relaxed); - do - { - size = std::min(end, config.evaluation_size); - begin = end - size; - } while (!thread_helper.next_gen_left.compare_exchange_weak( - end, end - size, std::memory_order::memory_order_relaxed, std::memory_order::memory_order_relaxed)); - - while (begin != end) - { - auto index = config.elites + begin; - tree_t& c1 = next_pop.get_individuals()[index].tree; - tree_t* c2 = nullptr; - if (begin + 1 < end) - c2 = &next_pop.get_individuals()[index + 1].tree; - begin += perform_selection(crossover_selection, mutation_selection, reproduction_selection, c1, c2); - } - } - } - thread_helper.barrier.wait(); - })); - thread_helper.thread_function_condition.notify_all(); - } - if (eval_fitness_now) - evaluate_fitness_internal(); - } - - template - void setup_steady_state_evaluation(FitnessFunc& fitness_function, SelectionStrat& replacement_strategy, size_t replacement_amount, - Crossover& crossover_selection, Mutation& mutation_selection, Reproduction& reproduction_selection, - const bool eval_fitness_now = true) - { - selection_probabilities.replacement_amount = replacement_amount; - if (config.threads == 1) - { - BLT_INFO("Starting steady state with single thread variant!"); - thread_execution_service = std::unique_ptr>(new std::function( - [this, &fitness_function, &replacement_strategy, &crossover_selection, &mutation_selection, &reproduction_selection](size_t) { - single_threaded_fitness_eval()(fitness_function); - - if (thread_helper.next_gen_left > 0) - { - current_stats.normalized_fitness.clear(); - double sum_of_prob = 0; - for (const auto& ind : current_pop) - { - const auto prob = (ind.fitness.adjusted_fitness / current_stats.overall_fitness); - current_stats.normalized_fitness.push_back(sum_of_prob + prob); - sum_of_prob += prob; - } - - next_pop = population_t(current_pop); - - replacement_strategy.pre_process(*this, next_pop); - crossover_selection.pre_process(*this, current_pop); - mutation_selection.pre_process(*this, current_pop); - reproduction_selection.pre_process(*this, current_pop); - - while (thread_helper.next_gen_left > 0) - { - tree_t& c1 = replacement_strategy.select(*this, next_pop); - tree_t* c2 = nullptr; - if (thread_helper.next_gen_left > 1) - while (c2 != &c1) - c2 = &replacement_strategy.select(*this, next_pop); - thread_helper.next_gen_left -= perform_selection(crossover_selection, mutation_selection, reproduction_selection, c1, - c2); - } - - thread_helper.next_gen_left = 0; - } - })); - } else - { - BLT_INFO("Starting steady state thread execution service!"); - std::scoped_lock lock(thread_helper.thread_function_control); - thread_execution_service = std::unique_ptr>(new std::function( - [this, &fitness_function, &replacement_strategy, &crossover_selection, &mutation_selection, &reproduction_selection]( - const size_t id) { - thread_helper.barrier.wait(); - - multi_threaded_fitness_eval()(fitness_function, id); - - if (thread_helper.next_gen_left > 0) - { - thread_helper.barrier.wait(); - if (id == 0) - { - current_stats.normalized_fitness.clear(); - double sum_of_prob = 0; - for (const auto& ind : current_pop) - { - const auto prob = (ind.fitness.adjusted_fitness / current_stats.overall_fitness); - current_stats.normalized_fitness.push_back(sum_of_prob + prob); - sum_of_prob += prob; - } - - current_pop = population_t(next_pop); - - replacement_strategy.pre_process(*this, next_pop); - crossover_selection.pre_process(*this, current_pop); - if (&crossover_selection != &mutation_selection) - mutation_selection.pre_process(*this, current_pop); - if (&crossover_selection != &reproduction_selection) - reproduction_selection.pre_process(*this, current_pop); - } - thread_helper.barrier.wait(); - - while (thread_helper.next_gen_left > 0) - { - size_t size = 0; - size_t end = thread_helper.next_gen_left.load(std::memory_order_relaxed); - do - { - size = std::min(end, config.evaluation_size); - } while (!thread_helper.next_gen_left.compare_exchange_weak( - end, end - size, std::memory_order::memory_order_relaxed, std::memory_order::memory_order_relaxed)); - - while (size > 0) - { - tree_t& c1 = replacement_strategy.select(*this, next_pop); - tree_t* c2 = nullptr; - if (thread_helper.next_gen_left > 1) - while (c2 != &c1) - c2 = &replacement_strategy.select(*this, next_pop); - size -= perform_selection(crossover_selection, mutation_selection, reproduction_selection, c1, c2); - } - } - } - thread_helper.barrier.wait(); - })); - thread_helper.thread_function_condition.notify_all(); - } - if (eval_fitness_now) - evaluate_fitness_internal(); - } - - [[nodiscard]] bool should_terminate() const - { - return current_generation >= config.max_generations || fitness_should_exit; - } - - [[nodiscard]] bool should_thread_terminate() const - { - return thread_helper.lifetime_over; - } - - operator_id select_terminal(const type_id id) - { - // we wanted a terminal, but could not find one, so we will select from a function that has a terminal - if (storage.terminals[id].empty()) - return select_non_terminal_too_deep(id); - return get_random().select(storage.terminals[id]); - } - - operator_id select_non_terminal(const type_id id) - { - // non-terminal doesn't exist, return a terminal. This is useful for types that are defined only to have a random value, nothing more. - // was considering an std::optional<> but that would complicate the generator code considerably. I'll mark this as a TODO for v2 - if (storage.non_terminals[id].empty()) - return select_terminal(id); - return get_random().select(storage.non_terminals[id]); - } - - operator_id select_non_terminal_too_deep(const type_id id) - { - // this should probably be an error. - if (storage.operators_ordered_terminals[id].empty()) - BLT_ABORT("An impossible state has been reached. Please consult the manual. Error 43"); - return get_random().select(storage.operators_ordered_terminals[id]).first; - } - - auto& get_current_pop() - { - return current_pop; - } - - [[nodiscard]] random_t& get_random() const; - - [[nodiscard]] const prog_config_t& get_config() const - { - return config; - } - - void set_config(const prog_config_t& config) - { - this->config = config; - selection_probabilities.update(this->config); - } - - [[nodiscard]] type_provider& get_typesystem() - { - return storage.system; - } - - [[nodiscard]] operator_info_t& get_operator_info(const operator_id id) - { - return storage.operators[id]; - } - - [[nodiscard]] detail::print_func_t& get_print_func(const operator_id id) - { - return storage.print_funcs[id]; - } - - [[nodiscard]] detail::destroy_func_t& get_destroy_func(const operator_id id) - { - return storage.destroy_funcs[id]; - } - - [[nodiscard]] std::optional get_name(const operator_id id) const - { - return storage.names[id]; - } - - [[nodiscard]] tracked_vector& get_type_terminals(const type_id id) - { - return storage.terminals[id]; - } - - [[nodiscard]] tracked_vector& get_type_non_terminals(const type_id id) - { - return storage.non_terminals[id]; - } - - [[nodiscard]] detail::eval_func_t& get_eval_func() - { - return storage.eval_func; - } - - [[nodiscard]] auto get_current_generation() const - { - return current_generation.load(); - } - - [[nodiscard]] const auto& get_population_stats() const - { - return current_stats; - } - - [[nodiscard]] bool is_operator_ephemeral(const operator_id id) const - { - return storage.operator_flags.find(static_cast(id))->second.is_ephemeral(); - } - - [[nodiscard]] bool operator_has_ephemeral_drop(const operator_id id) const - { - return storage.operator_flags.find(static_cast(id))->second.has_ephemeral_drop(); - } - - [[nodiscard]] operator_special_flags get_operator_flags(const operator_id id) const - { - return storage.operator_flags.find(static_cast(id))->second; - } - - void set_operations(program_operator_storage_t op) - { - storage = std::move(op); - } - - template - std::array get_best_indexes() - { - std::array arr; - - tracked_vector> values; - values.reserve(current_pop.get_individuals().size()); - - for (const auto& [index, value] : enumerate(current_pop.get_individuals())) - values.emplace_back(index, value.fitness.adjusted_fitness); - - std::sort(values.begin(), values.end(), [](const auto& a, const auto& b) { - return a.second > b.second; - }); - - for (size_t i = 0; i < std::min(size, config.population_size); ++i) - arr[i] = values[i].first; - for (size_t i = std::min(size, config.population_size); i < size; ++i) - arr[i] = 0; - - return arr; - } - - template - auto get_best_trees() - { - return convert_array, size>>(get_best_indexes(), - [this](auto&& arr, size_t index) -> tree_t& { - return current_pop.get_individuals()[arr[index]].tree; - }, std::make_integer_sequence()); - } - - template - auto get_best_individuals() - { - return convert_array, size>>(get_best_indexes(), - [this](auto&& arr, size_t index) -> individual_t& { - return current_pop.get_individuals()[arr[index]]; - }, std::make_integer_sequence()); - } - - void save_generation(fs::writer_t& writer); - - void save_state(fs::writer_t& writer); - - void load_generation(fs::reader_t& reader); - - void load_state(fs::reader_t& reader); - - private: - template - auto single_threaded_fitness_eval() - { - return [this](FitnessFunc& fitness_function) { - if (thread_helper.evaluation_left > 0) - { - current_stats.normalized_fitness.clear(); - double sum_of_prob = 0; - perform_fitness_function(0, current_pop.get_individuals().size(), fitness_function); - for (const auto& ind : current_pop) - { - const auto prob = (ind.fitness.adjusted_fitness / current_stats.overall_fitness); - current_stats.normalized_fitness.push_back(sum_of_prob + prob); - sum_of_prob += prob; - } - std::sort(current_pop.begin(), current_pop.end(), [](const auto& a, const auto& b) - { - return a.fitness.adjusted_fitness > b.fitness.adjusted_fitness; - }); - thread_helper.evaluation_left = 0; - } - }; - } - - template - auto multi_threaded_fitness_eval() - { - return [this](FitnessFunc& fitness_function, size_t thread_id) { - if (thread_helper.evaluation_left > 0) - { - thread_helper.barrier.wait(); - while (thread_helper.evaluation_left > 0) - { - size_t size = 0; - size_t begin = 0; - size_t end = thread_helper.evaluation_left.load(std::memory_order_relaxed); - do - { - size = std::min(end, config.evaluation_size); - begin = end - size; - } while (!thread_helper.evaluation_left.compare_exchange_weak(end, end - size, std::memory_order::memory_order_relaxed, - std::memory_order::memory_order_relaxed)); - perform_fitness_function(begin, end, fitness_function); - } - thread_helper.barrier.wait(); - if (thread_id == 0) - { - std::sort(current_pop.begin(), current_pop.end(), [](const auto& a, const auto& b) - { - return a.fitness.adjusted_fitness > b.fitness.adjusted_fitness; - }); - } - thread_helper.barrier.wait(); - } - }; - } - - template - void perform_fitness_function(const size_t begin, const size_t end, FitnessFunction& fitness_function) - { - using LambdaReturn = std::invoke_result_t; - for (size_t i = begin; i < end; i++) - { - auto& ind = current_pop.get_individuals()[i]; - - ind.fitness = {}; - if constexpr (std::is_same_v || std::is_convertible_v) - { - if (fitness_function(ind.tree, ind.fitness, i)) - fitness_should_exit = true; - } else - { - fitness_function(ind.tree, ind.fitness, i); - } - - auto old_best = current_stats.best_fitness.load(std::memory_order_relaxed); - while (ind.fitness.adjusted_fitness > old_best && !current_stats.best_fitness.compare_exchange_weak( - old_best, ind.fitness.adjusted_fitness, std::memory_order_relaxed, std::memory_order_relaxed)) - {} - - auto old_worst = current_stats.worst_fitness.load(std::memory_order_relaxed); - while (ind.fitness.adjusted_fitness < old_worst && !current_stats.worst_fitness.compare_exchange_weak( - old_worst, ind.fitness.adjusted_fitness, std::memory_order_relaxed, std::memory_order_relaxed)) - {} - - auto old_overall = current_stats.overall_fitness.load(std::memory_order_relaxed); - while (!current_stats.overall_fitness.compare_exchange_weak(old_overall, ind.fitness.adjusted_fitness + old_overall, - std::memory_order_relaxed, std::memory_order_relaxed)) - {} - } - } - - template - size_t perform_selection(Crossover& crossover, Mutation& mutation, Reproduction& reproduction, tree_t& c1, tree_t* c2) - { - if (get_random().choice(selection_probabilities.crossover_chance)) - { - auto ptr = c2; - if (ptr == nullptr) - ptr = &tree_t::get_thread_local(*this); - #ifdef BLT_TRACK_ALLOCATIONS +#endif + } + + void reset_program(type_id root_type, bool eval_fitness_now = true) + { + current_generation = 0; + current_pop = config.pop_initializer.get().generate({ + *this, root_type, config.population_size, config.initial_min_tree_size, config.initial_max_tree_size + }); + next_pop = population_t(current_pop); + BLT_ASSERT_MSG(current_pop.get_individuals().size() == config.population_size, + ("cur pop size: " + std::to_string(current_pop.get_individuals().size())).c_str()); + BLT_ASSERT_MSG(next_pop.get_individuals().size() == config.population_size, + ("next pop size: " + std::to_string(next_pop.get_individuals().size())).c_str()); + if (eval_fitness_now) + evaluate_fitness_internal(); + } + + void kill() + { + thread_helper.lifetime_over = true; + } + + void generate_initial_population(const type_id root_type) + { + current_pop = config.pop_initializer.get().generate({ + *this, root_type, config.population_size, config.initial_min_tree_size, config.initial_max_tree_size + }); + next_pop = population_t(current_pop); + BLT_ASSERT_MSG(current_pop.get_individuals().size() == config.population_size, + ("cur pop size: " + std::to_string(current_pop.get_individuals().size())).c_str()); + BLT_ASSERT_MSG(next_pop.get_individuals().size() == config.population_size, + ("next pop size: " + std::to_string(next_pop.get_individuals().size())).c_str()); + } + + /** + * takes in a reference to a function for the fitness evaluation function (must return a value convertable to double) + * The lambda must accept a tree for evaluation, and an index (current tree) + * + * tree_t& current_tree, blt::size_t index_of_tree + * + * Container must be concurrently accessible from multiple threads using operator[] + * + * NOTE: the larger the adjusted fitness, the better. + */ + template + void setup_generational_evaluation(FitnessFunc& fitness_function, Crossover& crossover_selection, Mutation& mutation_selection, + Reproduction& reproduction_selection, bool eval_fitness_now = true) + { + if (config.threads == 1) + { + BLT_INFO("Starting generational with single thread variant!"); + thread_execution_service = std::unique_ptr>(new std::function( + [this, &fitness_function, &crossover_selection, &mutation_selection, &reproduction_selection](size_t) + { + single_threaded_fitness_eval()(fitness_function); + + if (thread_helper.next_gen_left > 0) + { + current_stats.normalized_fitness.clear(); + double sum_of_prob = 0; + for (const auto& ind : current_pop) + { + const auto prob = (ind.fitness.adjusted_fitness / current_stats.overall_fitness); + current_stats.normalized_fitness.push_back(sum_of_prob + prob); + sum_of_prob += prob; + } + + const auto args = get_selector_args(); + + crossover_selection.pre_process(*this, current_pop); + mutation_selection.pre_process(*this, current_pop); + reproduction_selection.pre_process(*this, current_pop); + + size_t start = detail::perform_elitism(args, next_pop); + + while (start < config.population_size) + { + tree_t& c1 = next_pop.get_individuals()[start].tree; + tree_t* c2 = nullptr; + if (start + 1 < config.population_size) + c2 = &next_pop.get_individuals()[start + 1].tree; + start += perform_selection(crossover_selection, mutation_selection, reproduction_selection, c1, c2); + } + + thread_helper.next_gen_left = 0; + } + })); + } + else + { + BLT_INFO("Starting generational thread execution service!"); + std::scoped_lock lock(thread_helper.thread_function_control); + thread_execution_service = std::unique_ptr>(new std::function( + [this, &fitness_function, &crossover_selection, &mutation_selection, &reproduction_selection](const size_t id) + { + thread_helper.barrier.wait(); + + multi_threaded_fitness_eval()(fitness_function, id); + + if (thread_helper.next_gen_left > 0) + { + thread_helper.barrier.wait(); + auto args = get_selector_args(); + if (id == 0) + { + current_stats.normalized_fitness.clear(); + double sum_of_prob = 0; + for (const auto& ind : current_pop) + { + const auto prob = (ind.fitness.adjusted_fitness / current_stats.overall_fitness); + current_stats.normalized_fitness.push_back(sum_of_prob + prob); + sum_of_prob += prob; + } + + crossover_selection.pre_process(*this, current_pop); + if (&crossover_selection != &mutation_selection) + mutation_selection.pre_process(*this, current_pop); + if (&crossover_selection != &reproduction_selection) + reproduction_selection.pre_process(*this, current_pop); + const auto elite_amount = detail::perform_elitism(args, next_pop); + thread_helper.next_gen_left -= elite_amount; + } + thread_helper.barrier.wait(); + + while (thread_helper.next_gen_left > 0) + { + size_t size = 0; + size_t begin = 0; + size_t end = thread_helper.next_gen_left.load(std::memory_order_relaxed); + do + { + size = std::min(end, config.evaluation_size); + begin = end - size; + } + while (!thread_helper.next_gen_left.compare_exchange_weak( + end, end - size, std::memory_order::memory_order_relaxed, std::memory_order::memory_order_relaxed)); + + while (begin != end) + { + auto index = config.elites + begin; + tree_t& c1 = next_pop.get_individuals()[index].tree; + tree_t* c2 = nullptr; + if (begin + 1 < end) + c2 = &next_pop.get_individuals()[index + 1].tree; + begin += perform_selection(crossover_selection, mutation_selection, reproduction_selection, c1, c2); + } + } + } + thread_helper.barrier.wait(); + })); + thread_helper.thread_function_condition.notify_all(); + } + if (eval_fitness_now) + { + BLT_ASSERT( + !current_pop.get_individuals().empty() && current_pop.get_individuals().size() == config.population_size && + "Attempted to evaluate fitness but population was empty. Did you forget to call generate_initial_population()?" && + "You can also pass false to the function to prevent immediate fitness evaluation"); + evaluate_fitness_internal(); + } + } + + template + void setup_steady_state_evaluation(FitnessFunc& fitness_function, SelectionStrat& replacement_strategy, size_t replacement_amount, + Crossover& crossover_selection, Mutation& mutation_selection, Reproduction& reproduction_selection, + const bool eval_fitness_now = true) + { + selection_probabilities.replacement_amount = replacement_amount; + if (config.threads == 1) + { + BLT_INFO("Starting steady state with single thread variant!"); + thread_execution_service = std::unique_ptr>(new std::function( + [this, &fitness_function, &replacement_strategy, &crossover_selection, &mutation_selection, &reproduction_selection](size_t) + { + single_threaded_fitness_eval()(fitness_function); + + if (thread_helper.next_gen_left > 0) + { + current_stats.normalized_fitness.clear(); + double sum_of_prob = 0; + for (const auto& ind : current_pop) + { + const auto prob = (ind.fitness.adjusted_fitness / current_stats.overall_fitness); + current_stats.normalized_fitness.push_back(sum_of_prob + prob); + sum_of_prob += prob; + } + + next_pop = population_t(current_pop); + + replacement_strategy.pre_process(*this, next_pop); + crossover_selection.pre_process(*this, current_pop); + mutation_selection.pre_process(*this, current_pop); + reproduction_selection.pre_process(*this, current_pop); + + while (thread_helper.next_gen_left > 0) + { + tree_t& c1 = replacement_strategy.select(*this, next_pop); + tree_t* c2 = nullptr; + if (thread_helper.next_gen_left > 1) + while (c2 != &c1) + c2 = &replacement_strategy.select(*this, next_pop); + thread_helper.next_gen_left -= perform_selection(crossover_selection, mutation_selection, reproduction_selection, c1, + c2); + } + + thread_helper.next_gen_left = 0; + } + })); + } + else + { + BLT_INFO("Starting steady state thread execution service!"); + std::scoped_lock lock(thread_helper.thread_function_control); + thread_execution_service = std::unique_ptr>(new std::function( + [this, &fitness_function, &replacement_strategy, &crossover_selection, &mutation_selection, &reproduction_selection]( + const size_t id) + { + thread_helper.barrier.wait(); + + multi_threaded_fitness_eval()(fitness_function, id); + + if (thread_helper.next_gen_left > 0) + { + thread_helper.barrier.wait(); + if (id == 0) + { + current_stats.normalized_fitness.clear(); + double sum_of_prob = 0; + for (const auto& ind : current_pop) + { + const auto prob = (ind.fitness.adjusted_fitness / current_stats.overall_fitness); + current_stats.normalized_fitness.push_back(sum_of_prob + prob); + sum_of_prob += prob; + } + + current_pop = population_t(next_pop); + + replacement_strategy.pre_process(*this, next_pop); + crossover_selection.pre_process(*this, current_pop); + if (&crossover_selection != &mutation_selection) + mutation_selection.pre_process(*this, current_pop); + if (&crossover_selection != &reproduction_selection) + reproduction_selection.pre_process(*this, current_pop); + } + thread_helper.barrier.wait(); + + while (thread_helper.next_gen_left > 0) + { + size_t size = 0; + size_t end = thread_helper.next_gen_left.load(std::memory_order_relaxed); + do + { + size = std::min(end, config.evaluation_size); + } + while (!thread_helper.next_gen_left.compare_exchange_weak( + end, end - size, std::memory_order::memory_order_relaxed, std::memory_order::memory_order_relaxed)); + + while (size > 0) + { + tree_t& c1 = replacement_strategy.select(*this, next_pop); + tree_t* c2 = nullptr; + if (thread_helper.next_gen_left > 1) + while (c2 != &c1) + c2 = &replacement_strategy.select(*this, next_pop); + size -= perform_selection(crossover_selection, mutation_selection, reproduction_selection, c1, c2); + } + } + } + thread_helper.barrier.wait(); + })); + thread_helper.thread_function_condition.notify_all(); + } + if (eval_fitness_now) + { + BLT_ASSERT( + !current_pop.get_individuals().empty() && current_pop.get_individuals().size() == config.population_size && + "Attempted to evaluate fitness but population was empty. Did you forget to call generate_initial_population()?" && + "You can also pass false to the function to prevent immediate fitness evaluation"); + evaluate_fitness_internal(); + } + } + + [[nodiscard]] bool should_terminate() const + { + return current_generation >= config.max_generations || fitness_should_exit; + } + + [[nodiscard]] bool should_thread_terminate() const + { + return thread_helper.lifetime_over; + } + + operator_id select_terminal(const type_id id) + { + // we wanted a terminal, but could not find one, so we will select from a function that has a terminal + if (storage.terminals[id].empty()) + return select_non_terminal_too_deep(id); + return get_random().select(storage.terminals[id]); + } + + operator_id select_non_terminal(const type_id id) + { + // non-terminal doesn't exist, return a terminal. This is useful for types that are defined only to have a random value, nothing more. + // was considering an std::optional<> but that would complicate the generator code considerably. I'll mark this as a TODO for v2 + if (storage.non_terminals[id].empty()) + return select_terminal(id); + return get_random().select(storage.non_terminals[id]); + } + + operator_id select_non_terminal_too_deep(const type_id id) + { + // this should probably be an error. + if (storage.operators_ordered_terminals[id].empty()) + BLT_ABORT("An impossible state has been reached. Please consult the manual. Error 43"); + return get_random().select(storage.operators_ordered_terminals[id]).first; + } + + auto& get_current_pop() + { + return current_pop; + } + + [[nodiscard]] random_t& get_random() const; + + [[nodiscard]] const prog_config_t& get_config() const + { + return config; + } + + void set_config(const prog_config_t& config) + { + this->config = config; + selection_probabilities.update(this->config); + } + + [[nodiscard]] type_provider& get_typesystem() + { + return storage.system; + } + + [[nodiscard]] operator_info_t& get_operator_info(const operator_id id) + { + return storage.operators[id]; + } + + [[nodiscard]] detail::print_func_t& get_print_func(const operator_id id) + { + return storage.print_funcs[id]; + } + + [[nodiscard]] detail::destroy_func_t& get_destroy_func(const operator_id id) + { + return storage.destroy_funcs[id]; + } + + [[nodiscard]] std::optional get_name(const operator_id id) const + { + return storage.names[id]; + } + + [[nodiscard]] tracked_vector& get_type_terminals(const type_id id) + { + return storage.terminals[id]; + } + + [[nodiscard]] tracked_vector& get_type_non_terminals(const type_id id) + { + return storage.non_terminals[id]; + } + + [[nodiscard]] detail::eval_func_t& get_eval_func() + { + return storage.eval_func; + } + + [[nodiscard]] auto get_current_generation() const + { + return current_generation.load(); + } + + [[nodiscard]] const auto& get_population_stats() const + { + return current_stats; + } + + [[nodiscard]] bool is_operator_ephemeral(const operator_id id) const + { + return storage.operator_flags.find(static_cast(id))->second.is_ephemeral(); + } + + [[nodiscard]] bool operator_has_ephemeral_drop(const operator_id id) const + { + return storage.operator_flags.find(static_cast(id))->second.has_ephemeral_drop(); + } + + [[nodiscard]] operator_special_flags get_operator_flags(const operator_id id) const + { + return storage.operator_flags.find(static_cast(id))->second; + } + + void set_operations(program_operator_storage_t op) + { + storage = std::move(op); + } + + template + std::array get_best_indexes() + { + std::array arr; + + tracked_vector> values; + values.reserve(current_pop.get_individuals().size()); + + for (const auto& [index, value] : enumerate(current_pop.get_individuals())) + values.emplace_back(index, value.fitness.adjusted_fitness); + + std::sort(values.begin(), values.end(), [](const auto& a, const auto& b) + { + return a.second > b.second; + }); + + for (size_t i = 0; i < std::min(size, config.population_size); ++i) + arr[i] = values[i].first; + for (size_t i = std::min(size, config.population_size); i < size; ++i) + arr[i] = 0; + + return arr; + } + + template + auto get_best_trees() + { + return convert_array, size>>(get_best_indexes(), + [this](auto&& arr, size_t index) -> tree_t& + { + return current_pop.get_individuals()[arr[index]].tree; + }, std::make_integer_sequence()); + } + + template + auto get_best_individuals() + { + return convert_array, size>>(get_best_indexes(), + [this](auto&& arr, size_t index) -> individual_t& + { + return current_pop.get_individuals()[arr[index]]; + }, std::make_integer_sequence()); + } + + void save_generation(fs::writer_t& writer); + + void save_state(fs::writer_t& writer); + + void load_generation(fs::reader_t& reader); + + void load_state(fs::reader_t& reader); + + private: + template + auto single_threaded_fitness_eval() + { + return [this](FitnessFunc& fitness_function) + { + if (thread_helper.evaluation_left > 0) + { + current_stats.normalized_fitness.clear(); + double sum_of_prob = 0; + perform_fitness_function(0, current_pop.get_individuals().size(), fitness_function); + for (const auto& ind : current_pop) + { + const auto prob = (ind.fitness.adjusted_fitness / current_stats.overall_fitness); + current_stats.normalized_fitness.push_back(sum_of_prob + prob); + sum_of_prob += prob; + } + std::sort(current_pop.begin(), current_pop.end(), [](const auto& a, const auto& b) + { + return a.fitness.adjusted_fitness > b.fitness.adjusted_fitness; + }); + thread_helper.evaluation_left = 0; + } + }; + } + + template + auto multi_threaded_fitness_eval() + { + return [this](FitnessFunc& fitness_function, size_t thread_id) + { + if (thread_helper.evaluation_left > 0) + { + thread_helper.barrier.wait(); + while (thread_helper.evaluation_left > 0) + { + size_t size = 0; + size_t begin = 0; + size_t end = thread_helper.evaluation_left.load(std::memory_order_relaxed); + do + { + size = std::min(end, config.evaluation_size); + begin = end - size; + } + while (!thread_helper.evaluation_left.compare_exchange_weak(end, end - size, std::memory_order::memory_order_relaxed, + std::memory_order::memory_order_relaxed)); + perform_fitness_function(begin, end, fitness_function); + } + thread_helper.barrier.wait(); + if (thread_id == 0) + { + std::sort(current_pop.begin(), current_pop.end(), [](const auto& a, const auto& b) + { + return a.fitness.adjusted_fitness > b.fitness.adjusted_fitness; + }); + } + thread_helper.barrier.wait(); + } + }; + } + + template + void perform_fitness_function(const size_t begin, const size_t end, FitnessFunction& fitness_function) + { + using LambdaReturn = std::invoke_result_t; + for (size_t i = begin; i < end; i++) + { + auto& ind = current_pop.get_individuals()[i]; + + ind.fitness = {}; + if constexpr (std::is_same_v || std::is_convertible_v) + { + if (fitness_function(ind.tree, ind.fitness, i)) + fitness_should_exit = true; + } + else + { + fitness_function(ind.tree, ind.fitness, i); + } + + auto old_best = current_stats.best_fitness.load(std::memory_order_relaxed); + while (ind.fitness.adjusted_fitness > old_best && !current_stats.best_fitness.compare_exchange_weak( + old_best, ind.fitness.adjusted_fitness, std::memory_order_relaxed, std::memory_order_relaxed)) + { + } + + auto old_worst = current_stats.worst_fitness.load(std::memory_order_relaxed); + while (ind.fitness.adjusted_fitness < old_worst && !current_stats.worst_fitness.compare_exchange_weak( + old_worst, ind.fitness.adjusted_fitness, std::memory_order_relaxed, std::memory_order_relaxed)) + { + } + + auto old_overall = current_stats.overall_fitness.load(std::memory_order_relaxed); + while (!current_stats.overall_fitness.compare_exchange_weak(old_overall, ind.fitness.adjusted_fitness + old_overall, + std::memory_order_relaxed, std::memory_order_relaxed)) + { + } + } + } + + template + size_t perform_selection(Crossover& crossover, Mutation& mutation, Reproduction& reproduction, tree_t& c1, tree_t* c2) + { + if (get_random().choice(selection_probabilities.crossover_chance)) + { + auto ptr = c2; + if (ptr == nullptr) + ptr = &tree_t::get_thread_local(*this); +#ifdef BLT_TRACK_ALLOCATIONS auto state = tracker.start_measurement_thread_local(); - #endif - const tree_t* p1; - const tree_t* p2; - size_t runs = 0; - do - { - // BLT_TRACE("%lu %p %p", runs, &c1, &tree); - p1 = &crossover.select(*this, current_pop); - p2 = &crossover.select(*this, current_pop); - // BLT_TRACE("%p %p || %lu", p1, p2, current_pop.get_individuals().size()); +#endif + const tree_t* p1; + const tree_t* p2; + size_t runs = 0; + do + { + // BLT_TRACE("%lu %p %p", runs, &c1, &tree); + p1 = &crossover.select(*this, current_pop); + p2 = &crossover.select(*this, current_pop); + // BLT_TRACE("%p %p || %lu", p1, p2, current_pop.get_individuals().size()); - c1.copy_fast(*p1); - ptr->copy_fast(*p2); - // ptr->copy_fast(*p2); + c1.copy_fast(*p1); + ptr->copy_fast(*p2); + // ptr->copy_fast(*p2); - if (++runs >= config.crossover.get().get_config().max_crossover_iterations) - return 0; - #ifdef BLT_TRACK_ALLOCATIONS + if (++runs >= config.crossover.get().get_config().max_crossover_iterations) + return 0; +#ifdef BLT_TRACK_ALLOCATIONS crossover_calls.value(1); - #endif - } while (!config.crossover.get().apply(*this, *p1, *p2, c1, *ptr)); - #ifdef BLT_TRACK_ALLOCATIONS +#endif + } + while (!config.crossover.get().apply(*this, *p1, *p2, c1, *ptr)); +#ifdef BLT_TRACK_ALLOCATIONS tracker.stop_measurement_thread_local(state); crossover_calls.call(); if (state.getAllocatedByteDifference() != 0) @@ -934,30 +972,31 @@ namespace blt::gp crossover_allocations.call(state.getAllocatedByteDifference()); crossover_allocations.set_value(std::max(crossover_allocations.get_value(), state.getAllocatedByteDifference())); } - #endif - if (c2 == nullptr) - { - tree_t::get_thread_local(*this).clear(*this); - return 1; - } - return 2; - } - if (get_random().choice(selection_probabilities.mutation_chance)) - { - #ifdef BLT_TRACK_ALLOCATIONS +#endif + if (c2 == nullptr) + { + tree_t::get_thread_local(*this).clear(*this); + return 1; + } + return 2; + } + if (get_random().choice(selection_probabilities.mutation_chance)) + { +#ifdef BLT_TRACK_ALLOCATIONS auto state = tracker.start_measurement_thread_local(); - #endif - // mutation - const tree_t* p; - do - { - p = &mutation.select(*this, current_pop); - c1.copy_fast(*p); - #ifdef BLT_TRACK_ALLOCATIONS +#endif + // mutation + const tree_t* p; + do + { + p = &mutation.select(*this, current_pop); + c1.copy_fast(*p); +#ifdef BLT_TRACK_ALLOCATIONS mutation_calls.value(1); - #endif - } while (!config.mutator.get().apply(*this, *p, c1)); - #ifdef BLT_TRACK_ALLOCATIONS +#endif + } + while (!config.mutator.get().apply(*this, *p, c1)); +#ifdef BLT_TRACK_ALLOCATIONS tracker.stop_measurement_thread_local(state); mutation_calls.call(); if (state.getAllocationDifference() != 0) @@ -965,17 +1004,17 @@ namespace blt::gp mutation_allocations.call(state.getAllocatedByteDifference()); mutation_allocations.set_value(std::max(mutation_allocations.get_value(), state.getAllocatedByteDifference())); } - #endif - return 1; - } - if (selection_probabilities.reproduction_chance > 0) - { - #ifdef BLT_TRACK_ALLOCATIONS +#endif + return 1; + } + if (selection_probabilities.reproduction_chance > 0) + { +#ifdef BLT_TRACK_ALLOCATIONS auto state = tracker.start_measurement_thread_local(); - #endif - // reproduction - c1.copy_fast(reproduction.select(*this, current_pop)); - #ifdef BLT_TRACK_ALLOCATIONS +#endif + // reproduction + c1.copy_fast(reproduction.select(*this, current_pop)); +#ifdef BLT_TRACK_ALLOCATIONS tracker.stop_measurement_thread_local(state); reproduction_calls.call(); reproduction_calls.value(1); @@ -984,88 +1023,89 @@ namespace blt::gp reproduction_allocations.call(state.getAllocatedByteDifference()); reproduction_allocations.set_value(std::max(reproduction_allocations.get_value(), state.getAllocatedByteDifference())); } - #endif - return 1; - } +#endif + return 1; + } - return 0; - } + return 0; + } - selector_args get_selector_args() - { - return {*this, current_pop, current_stats, config, get_random()}; - } + selector_args get_selector_args() + { + return {*this, current_pop, current_stats, config, get_random()}; + } - template - Return convert_array(std::array&& arr, Accessor&& accessor, std::integer_sequence) - { - return Return{accessor(arr, indexes)...}; - } + template + Return convert_array(std::array&& arr, Accessor&& accessor, std::integer_sequence) + { + return Return{accessor(arr, indexes)...}; + } - void create_threads(); + void create_threads(); - void evaluate_fitness_internal() - { - statistic_history.push_back(current_stats); - current_stats.clear(); - thread_helper.evaluation_left.store(config.population_size, std::memory_order_release); - (*thread_execution_service)(0); + void evaluate_fitness_internal() + { + statistic_history.push_back(current_stats); + current_stats.clear(); + thread_helper.evaluation_left.store(config.population_size, std::memory_order_release); + (*thread_execution_service)(0); - current_stats.average_fitness = current_stats.overall_fitness / static_cast(config.population_size); - } + current_stats.average_fitness = current_stats.overall_fitness / static_cast(config.population_size); + } - private: - program_operator_storage_t storage; - std::function seed_func; - prog_config_t config{}; + private: + program_operator_storage_t storage; + std::function seed_func; + prog_config_t config{}; - // internal cache which stores already calculated probability values - struct - { - double crossover_chance = 0; - double mutation_chance = 0; - double reproduction_chance = 0; + // internal cache which stores already calculated probability values + struct + { + double crossover_chance = 0; + double mutation_chance = 0; + double reproduction_chance = 0; - std::optional replacement_amount; + std::optional replacement_amount; - void update(const prog_config_t& config) - { - const auto total = config.crossover_chance + config.mutation_chance + config.reproduction_chance; - crossover_chance = config.crossover_chance / total; - mutation_chance = config.mutation_chance / total; - reproduction_chance = config.reproduction_chance / total; - } - } selection_probabilities; + void update(const prog_config_t& config) + { + const auto total = config.crossover_chance + config.mutation_chance + config.reproduction_chance; + crossover_chance = config.crossover_chance / total; + mutation_chance = config.mutation_chance / total; + reproduction_chance = config.reproduction_chance / total; + } + } selection_probabilities; - population_t current_pop; - population_t next_pop; + population_t current_pop; + population_t next_pop; - std::atomic_uint64_t current_generation = 0; + std::atomic_uint64_t current_generation = 0; - std::atomic_bool fitness_should_exit = false; + std::atomic_bool fitness_should_exit = false; - population_stats current_stats{}; - tracked_vector statistic_history; + population_stats current_stats{}; + tracked_vector statistic_history; - struct concurrency_storage - { - std::vector> threads; + struct concurrency_storage + { + std::vector> threads; - std::mutex thread_function_control{}; - std::condition_variable thread_function_condition{}; + std::mutex thread_function_control{}; + std::condition_variable thread_function_condition{}; - std::atomic_uint64_t evaluation_left = 0; - std::atomic_uint64_t next_gen_left = 0; + std::atomic_uint64_t evaluation_left = 0; + std::atomic_uint64_t next_gen_left = 0; - std::atomic_bool lifetime_over = false; - blt::barrier_t barrier; + std::atomic_bool lifetime_over = false; + blt::barrier_t barrier; - explicit concurrency_storage(blt::size_t threads): barrier(threads, lifetime_over) - {} - } thread_helper{config.threads == 0 ? std::thread::hardware_concurrency() : config.threads}; + explicit concurrency_storage(blt::size_t threads): barrier(threads, lifetime_over) + { + } + } thread_helper{config.threads == 0 ? std::thread::hardware_concurrency() : config.threads}; - std::unique_ptr> thread_execution_service = nullptr; - }; + std::unique_ptr> thread_execution_service = nullptr; + }; } #endif //BLT_GP_PROGRAM_H diff --git a/include/blt/gp/tree.h b/include/blt/gp/tree.h index f9a4be5..263e0af 100644 --- a/include/blt/gp/tree.h +++ b/include/blt/gp/tree.h @@ -88,6 +88,8 @@ namespace blt::gp return m_flags; } + friend bool operator==(const op_container_t& a, const op_container_t& b); + private: size_t m_type_size; operator_id m_id; @@ -625,6 +627,13 @@ namespace blt::gp static tree_t& get_thread_local(gp_program& program); + friend bool operator==(const tree_t& a, const tree_t& b); + + friend bool operator!=(const tree_t& a, const tree_t& b) + { + return !(a == b); + } + private: void handle_operator_inserted(const op_container_t& op); @@ -755,6 +764,13 @@ namespace blt::gp individual_t& operator=(const individual_t&) = delete; individual_t& operator=(individual_t&&) = default; + + friend bool operator==(const individual_t& a, const individual_t& b); + + friend bool operator!=(const individual_t& a, const individual_t& b) + { + return !(a == b); + } }; class population_t diff --git a/src/tree.cpp b/src/tree.cpp index 5f7a78b..b59a2d6 100644 --- a/src/tree.cpp +++ b/src/tree.cpp @@ -717,12 +717,12 @@ namespace blt::gp operator_id id; std::memcpy(&id, in, sizeof(operator_id)); in += sizeof(operator_id); - operations.push_back({ + operations.emplace_back( m_program->get_typesystem().get_type(m_program->get_operator_info(id).return_type).size(), id, m_program->is_operator_ephemeral(id), m_program->get_operator_flags(id) - }); + ); } size_t val_size; std::memcpy(&val_size, in, sizeof(size_t)); @@ -740,17 +740,17 @@ namespace blt::gp { operator_id id; BLT_ASSERT(file.read(&id, sizeof(operator_id)) == sizeof(operator_id)); - operations.push_back({ + operations.emplace_back( m_program->get_typesystem().get_type(m_program->get_operator_info(id).return_type).size(), id, m_program->is_operator_ephemeral(id), m_program->get_operator_flags(id) - }); + ); } - size_t val_size; - BLT_ASSERT(file.read(&val_size, sizeof(size_t)) == sizeof(size_t)); - values.resize(val_size); - BLT_ASSERT(file.read(values.data(), val_size) == val_size); + size_t bytes_in_head; + BLT_ASSERT(file.read(&bytes_in_head, sizeof(size_t)) == sizeof(size_t)); + values.resize(bytes_in_head); + BLT_ASSERT(file.read(values.data(), bytes_in_head) == bytes_in_head); } void tree_t::modify_operator(const size_t point, operator_id new_id, std::optional return_type) @@ -787,4 +787,23 @@ namespace blt::gp handle_operator_inserted(operations[point]); } } + + bool operator==(const tree_t& a, const tree_t& b) + { + if (a.operations.size() != b.operations.size()) + return false; + if (a.values.bytes_in_head() != b.values.bytes_in_head()) + return false; + return std::equal(a.operations.begin(), a.operations.end(), b.operations.begin()); + } + + bool operator==(const op_container_t& a, const op_container_t& b) + { + return a.id() == b.id(); + } + + bool operator==(const individual_t& a, const individual_t& b) + { + return a.tree == b.tree; + } } diff --git a/tests/serialization_test.cpp b/tests/serialization_test.cpp index 3512c56..c101fa8 100644 --- a/tests/serialization_test.cpp +++ b/tests/serialization_test.cpp @@ -15,9 +15,15 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ +#include + #include "../examples/symbolic_regression.h" #include #include +#include +#include +#include +#include using namespace blt::gp; @@ -35,7 +41,7 @@ prog_config_t config = prog_config_t() .set_reproduction_chance(0.1) .set_max_generations(50) .set_pop_size(500) - .set_thread_count(1); + .set_thread_count(0); example::symbolic_regression_t regression{691ul, config}; @@ -84,11 +90,16 @@ bool fitness_function(const tree_t& current_tree, fitness_t& fitness, size_t) int main() { operator_builder builder{}; - builder.build(addf, subf, mulf, pro_divf, op_sinf, op_cosf, op_expf, op_logf, litf, op_xf); - regression.get_program().set_operations(builder.grab()); + const auto& operators = builder.build(addf, subf, mulf, pro_divf, op_sinf, op_cosf, op_expf, op_logf, litf, op_xf); + regression.get_program().set_operations(operators); auto& program = regression.get_program(); static auto sel = select_tournament_t{}; + + gp_program test_program{691}; + test_program.set_operations(operators); + test_program.setup_generational_evaluation(fitness_function, sel, sel, sel, false); + program.generate_initial_population(program.get_typesystem().get_type().id()); program.setup_generational_evaluation(fitness_function, sel, sel, sel); while (!program.should_terminate()) @@ -100,5 +111,25 @@ int main() program.next_generation(); BLT_TRACE("Evaluate Fitness"); program.evaluate_fitness(); + { + std::filesystem::remove("serialization_test.data"); + std::ofstream stream{"serialization_test.data", std::ios::binary}; + blt::fs::fstream_writer_t writer{stream}; + program.save_generation(writer); + } + { + std::ifstream stream{"serialization_test.data", std::ios::binary}; + blt::fs::fstream_reader_t reader{stream}; + test_program.load_generation(reader); + } + // do a quick validation check + for (const auto& [saved, loaded] : blt::zip(program.get_current_pop(), test_program.get_current_pop())) + { + if (saved.tree != loaded.tree) + { + BLT_ERROR("Serializer Failed to correctly serialize tree to disk, trees are not equal!"); + std::exit(1); + } + } } } From b9c535f6c9bce2594dfff4d20d5c2d15eb97aa38 Mon Sep 17 00:00:00 2001 From: Brett Laptop Date: Thu, 17 Apr 2025 21:12:17 -0400 Subject: [PATCH 14/32] boy --- CMakeLists.txt | 2 +- lib/blt | 2 +- src/program.cpp | 40 ++++++++++++++++++++-------------------- src/tree.cpp | 2 +- 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3a9ff7b..a1cf2a1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -27,7 +27,7 @@ macro(compile_options target_name) sanitizers(${target_name}) endmacro() -project(blt-gp VERSION 0.5.9) +project(blt-gp VERSION 0.5.10) include(CTest) diff --git a/lib/blt b/lib/blt index 90cf177..a1bc8cf 160000 --- a/lib/blt +++ b/lib/blt @@ -1 +1 @@ -Subproject commit 90cf177c57f542ecdd132eaa6af0f1945ce8fe96 +Subproject commit a1bc8cf1c2390e507be6482dde0960daf9f662d8 diff --git a/src/program.cpp b/src/program.cpp index f6aea3b..40ad5b5 100644 --- a/src/program.cpp +++ b/src/program.cpp @@ -104,15 +104,15 @@ namespace blt::gp void load_stat(fs::reader_t& reader, population_stats& stat) { - reader.read(&stat.overall_fitness, sizeof(stat.overall_fitness)); - reader.read(&stat.average_fitness, sizeof(stat.average_fitness)); - reader.read(&stat.best_fitness, sizeof(stat.best_fitness)); - reader.read(&stat.worst_fitness, sizeof(stat.worst_fitness)); + BLT_ASSERT(reader.read(&stat.overall_fitness, sizeof(stat.overall_fitness)) == sizeof(stat.overall_fitness)); + BLT_ASSERT(reader.read(&stat.average_fitness, sizeof(stat.average_fitness)) == sizeof(stat.average_fitness)); + BLT_ASSERT(reader.read(&stat.best_fitness, sizeof(stat.best_fitness)) == sizeof(stat.best_fitness)); + BLT_ASSERT(reader.read(&stat.worst_fitness, sizeof(stat.worst_fitness)) == sizeof(stat.worst_fitness)); size_t fitness_count; - reader.read(&fitness_count, sizeof(fitness_count)); + BLT_ASSERT(reader.read(&fitness_count, sizeof(fitness_count)) == sizeof(size_t)); stat.normalized_fitness.resize(fitness_count); for (auto& fitness : stat.normalized_fitness) - reader.read(&fitness, sizeof(fitness)); + BLT_ASSERT(reader.read(&fitness, sizeof(fitness)) == sizeof(fitness)); } void gp_program::save_state(fs::writer_t& writer) @@ -150,37 +150,37 @@ namespace blt::gp void gp_program::load_state(fs::reader_t& reader) { size_t operator_count; - reader.read(&operator_count, sizeof(operator_count)); + BLT_ASSERT(reader.read(&operator_count, sizeof(operator_count)) == sizeof(operator_count)); if (operator_count != storage.operators.size()) throw std::runtime_error( "Invalid number of operators. Expected " + std::to_string(storage.operators.size()) + " found " + std::to_string(operator_count)); for (size_t i = 0; i < operator_count; i++) { size_t expected_i; - reader.read(&expected_i, sizeof(expected_i)); + BLT_ASSERT(reader.read(&expected_i, sizeof(expected_i)) == sizeof(expected_i)); if (expected_i != i) throw std::runtime_error("Loaded invalid operator ID. Expected " + std::to_string(i) + " found " + std::to_string(expected_i)); bool has_name; - reader.read(&has_name, sizeof(has_name)); + BLT_ASSERT(reader.read(&has_name, sizeof(has_name)) == sizeof(has_name)); if (has_name) { size_t size; - reader.read(&size, sizeof(size)); + BLT_ASSERT(reader.read(&size, sizeof(size)) == sizeof(size)); std::string name; name.resize(size); - reader.read(name.data(), size); + BLT_ASSERT(reader.read(name.data(), size) == size); if (!storage.names[i].has_value()) throw std::runtime_error("Expected operator ID " + std::to_string(i) + " to have name " + name); if (name != *storage.names[i]) throw std::runtime_error( "Operator ID " + std::to_string(i) + " expected to be named " + name + " found " + std::string(*storage.names[i])); - auto& op = storage.operators[i]; - auto& op_meta = storage.operator_metadata[i]; + const auto& op = storage.operators[i]; + const auto& op_meta = storage.operator_metadata[i]; decltype(std::declval().arg_size_bytes) arg_size_bytes; decltype(std::declval().return_size_bytes) return_size_bytes; - reader.read(&arg_size_bytes, sizeof(arg_size_bytes)); - reader.read(&return_size_bytes, sizeof(return_size_bytes)); + BLT_ASSERT(reader.read(&arg_size_bytes, sizeof(arg_size_bytes)) == sizeof(arg_size_bytes)); + BLT_ASSERT(reader.read(&return_size_bytes, sizeof(return_size_bytes)) == sizeof(return_size_bytes)); if (op_meta.arg_size_bytes != arg_size_bytes) throw std::runtime_error( @@ -194,7 +194,7 @@ namespace blt::gp std::to_string(return_size_bytes)); argc_t argc; - reader.read(&argc, sizeof(argc)); + BLT_ASSERT(reader.read(&argc, sizeof(argc)) == sizeof(argc)); if (argc.argc != op.argc.argc) throw std::runtime_error( "Operator ID " + std::to_string(i) + " expected " + std::to_string(op.argc.argc) + " arguments but got " + std::to_string( @@ -204,7 +204,7 @@ namespace blt::gp "Operator ID " + std::to_string(i) + " expected " + std::to_string(op.argc.argc_context) + " arguments but got " + std::to_string(argc.argc_context)); type_id return_type; - reader.read(&return_type, sizeof(return_type)); + BLT_ASSERT(reader.read(&return_type, sizeof(return_type)) == sizeof(return_type)); if (return_type != op.return_type) throw std::runtime_error( "Operator ID " + std::to_string(i) + " expected return type " + std::to_string(op.return_type) + " but got " + std::to_string( @@ -214,11 +214,11 @@ namespace blt::gp throw std::runtime_error( "Operator ID " + std::to_string(i) + " expected " + std::to_string(op.argument_types.size()) + " arguments but got " + std::to_string(arg_type_count)); - reader.read(&arg_type_count, sizeof(arg_type_count)); + BLT_ASSERT(reader.read(&arg_type_count, sizeof(arg_type_count)) == sizeof(return_type)); for (size_t j = 0; j < arg_type_count; j++) { type_id type; - reader.read(&type, sizeof(type)); + BLT_ASSERT(reader.read(&type, sizeof(type)) == sizeof(type)); if (type != op.argument_types[j]) throw std::runtime_error( "Operator ID " + std::to_string(i) + " expected argument " + std::to_string(j) + " to be of type " + std::to_string( @@ -227,7 +227,7 @@ namespace blt::gp } } size_t history_count; - reader.read(&history_count, sizeof(history_count)); + BLT_ASSERT(reader.read(&history_count, sizeof(history_count)) == sizeof(history_count)); statistic_history.resize(history_count); for (size_t i = 0; i < history_count; i++) load_stat(reader, statistic_history[i]); diff --git a/src/tree.cpp b/src/tree.cpp index b59a2d6..6513473 100644 --- a/src/tree.cpp +++ b/src/tree.cpp @@ -750,7 +750,7 @@ namespace blt::gp size_t bytes_in_head; BLT_ASSERT(file.read(&bytes_in_head, sizeof(size_t)) == sizeof(size_t)); values.resize(bytes_in_head); - BLT_ASSERT(file.read(values.data(), bytes_in_head) == bytes_in_head); + BLT_ASSERT(file.read(values.data(), bytes_in_head) == static_cast(bytes_in_head)); } void tree_t::modify_operator(const size_t point, operator_id new_id, std::optional return_type) From 487f7713772662d37d4e1c088c399183bea6ca86 Mon Sep 17 00:00:00 2001 From: Brett Date: Fri, 18 Apr 2025 13:34:32 -0400 Subject: [PATCH 15/32] working on transformers --- .idea/editor.xml | 2 + CMakeLists.txt | 2 +- include/blt/gp/program.h | 5 +++ include/blt/gp/transformers.h | 8 ++++ include/blt/gp/util/statistics.h | 14 +++++++ src/program.cpp | 4 +- src/transformers.cpp | 66 +++++++++++++++++++++++++------- tests/serialization_test.cpp | 39 ++++++++++++++++++- 8 files changed, 121 insertions(+), 19 deletions(-) diff --git a/.idea/editor.xml b/.idea/editor.xml index 6df7d16..5603a5a 100644 --- a/.idea/editor.xml +++ b/.idea/editor.xml @@ -5,6 +5,8 @@