From 809305dbbd9a4cda2fcc01429476552f3ee2ac42 Mon Sep 17 00:00:00 2001 From: mcoury Date: Sat, 10 Dec 2022 16:43:55 -0800 Subject: [PATCH 1/3] Added taskflow support --- .gitmodules | 3 + .../preprocessing/include/dwave/presolve.hpp | 175 ++++++++++++------ extern/taskflow | 1 + setup.py | 2 + testscpp/Makefile | 14 +- 5 files changed, 137 insertions(+), 58 deletions(-) create mode 160000 extern/taskflow diff --git a/.gitmodules b/.gitmodules index 51341cc..3747830 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,3 +2,6 @@ path = testscpp/Catch2 url = https://github.com/catchorg/Catch2.git branch = v2.x +[submodule "extern/taskflow"] + path = extern/taskflow + url = https://github.com/taskflow/taskflow.git diff --git a/dwave/preprocessing/include/dwave/presolve.hpp b/dwave/preprocessing/include/dwave/presolve.hpp index d6d63c1..e0bc4ab 100644 --- a/dwave/preprocessing/include/dwave/presolve.hpp +++ b/dwave/preprocessing/include/dwave/presolve.hpp @@ -19,6 +19,8 @@ #include #include +#include "taskflow/core/taskflow.hpp" +#include "taskflow/taskflow.hpp" #include "dimod/constrained_quadratic_model.h" namespace dwave { @@ -139,7 +141,7 @@ class Presolver { model_type detach_model(); /// Load the default presolve techniques. - void load_default_presolvers(); + void load_default_presolvers(int max_rounds = 100); /// Return a const reference to the held constrained quadratic model. const model_type& model() const; @@ -148,12 +150,29 @@ class Presolver { const Postsolver& postsolver() const; private: + struct TfHelper { + tf::Executor executor; + tf::Taskflow taskflow_onetime; + tf::Taskflow taskflow_trivial; + tf::Taskflow taskflow_cleanup; + int loop_counter; + bool loop_changed; + bool model_feasible = true; + + bool operator=(const struct TfHelper& that) { + return true; + } + }; + + struct TfHelper tf_helper_; + + void load_taskflow_onetime(); + void load_taskflow_trivial(int max_rounds = 100); + void load_taskflow_cleanup(); + model_type model_; Postsolver postsolver_; - // todo: replace this with a vector of pointers or similar - bool default_techniques_; - bool detached_; void substitute_self_loops_expr(dimod::Expression& expression, @@ -281,20 +300,17 @@ class Presolver { switch (constraint.sense()) { case dimod::Sense::EQ: if (constraint.offset() != constraint.rhs()) { - // need this exact message for Python - throw std::logic_error("infeasible"); + tf_helper_.model_feasible = false; } break; case dimod::Sense::LE: if (constraint.offset() > constraint.rhs()) { - // need this exact message for Python - throw std::logic_error("infeasible"); + tf_helper_.model_feasible = false; } break; case dimod::Sense::GE: if (constraint.offset() < constraint.rhs()) { - // need this exact message for Python - throw std::logic_error("infeasible"); + tf_helper_.model_feasible = false; } break; } @@ -418,54 +434,25 @@ class Presolver { template Presolver::Presolver() - : model_(), postsolver_(), default_techniques_(false), detached_(false) {} + : model_(), postsolver_(), detached_(false) {} template Presolver::Presolver(model_type model) - : model_(std::move(model)), postsolver_(), default_techniques_(), detached_(false) {} + : model_(std::move(model)), postsolver_(), detached_(false) {} template void Presolver::apply() { if (detached_) throw std::logic_error("model has been detached, presolver is no longer valid"); - // If no techniques have been loaded, return early. - if (!default_techniques_) return; - - // One time techniques ---------------------------------------------------- - - // *-- spin-to-binary - technique_spin_to_binary(); - // *-- remove offsets - technique_remove_offsets(); - // *-- flip >= constraints - technique_flip_constraints(); - // *-- remove self-loops - technique_remove_self_loops(); - - // Trivial techniques ----------------------------------------------------- - - bool changes = true; - const index_type max_num_rounds = 100; // todo: make configurable - for (index_type num_rounds = 0; num_rounds < max_num_rounds; ++num_rounds) { - if (!changes) break; - changes = false; - - // *-- clear out 0 variables/interactions in the constraints and objective - changes |= technique_remove_zero_biases(); - // *-- todo: check for NAN - changes |= technique_check_for_nan(); - // *-- remove single variable constraints - changes |= technique_remove_single_variable_constraints(); - // *-- tighten bounds based on vartype - changes |= technique_tighten_bounds(); - // *-- remove variables that are fixed by bounds - changes |= technique_remove_fixed_variables(); - } - - // Cleanup - - // *-- remove any invalid discrete markers - technique_remove_invalid_markers(); + tf_helper_.executor.run(tf_helper_.taskflow_onetime).wait(); + tf_helper_.executor.run(tf_helper_.taskflow_trivial).wait(); + if(tf_helper_.model_feasible) { + tf_helper_.executor.run(tf_helper_.taskflow_cleanup).wait(); + } + else { + // need this exact message for Python + throw std::logic_error("infeasible"); + } } template @@ -482,8 +469,92 @@ Presolver::detach_model() { } template -void Presolver::load_default_presolvers() { - default_techniques_ = true; +void Presolver::load_default_presolvers(int max_rounds) { + load_taskflow_onetime(); + load_taskflow_trivial(); + load_taskflow_cleanup(); +} + +template +void Presolver::load_taskflow_onetime() { + auto [a, b, c, d] = tf_helper_.taskflow_onetime.emplace( + [&]() { technique_spin_to_binary(); }, + [&]() { technique_remove_offsets(); }, + [&]() { technique_flip_constraints(); }, + [&]() { technique_remove_self_loops(); } + ); + + a.name("spin_to_binary"); + b.name("remove_offsets"); + c.name("flip_constraints"); + d.name("remove_self_loops"); + + a.precede(b); + b.precede(c); + c.precede(d); +} + +template +void Presolver::load_taskflow_trivial(int max_rounds) { + auto alpha = tf_helper_.taskflow_trivial.emplace( + [&]() { + tf_helper_.loop_changed = false; + tf_helper_.loop_counter = 0; + } + ); + auto [a, b, c, d, e] = tf_helper_.taskflow_trivial.emplace( + [&]() { tf_helper_.loop_changed |= technique_remove_zero_biases(); }, + [&]() { tf_helper_.loop_changed |= technique_check_for_nan(); }, + [&]() { tf_helper_.loop_changed |= technique_remove_single_variable_constraints(); }, + [&]() + { + if(tf_helper_.model_feasible) { + tf_helper_.loop_changed |= technique_tighten_bounds(); + } + }, + [&]() + { + if(tf_helper_.model_feasible) { + tf_helper_.loop_changed |= technique_remove_fixed_variables(); + } + } + ); + auto omega = tf_helper_.taskflow_trivial.emplace( + [&]() { + if(tf_helper_.model_feasible + && tf_helper_.loop_changed + && ++tf_helper_.loop_counter < max_rounds + ) { + tf_helper_.loop_changed = false; + return 0; // This will take us back to (a) + } + return 1; // This will cause us to exit + } + ); + + alpha.name("initialize"); + a.name("remove_zero_biases"); + b.name("check_for_nan"); + c.name("remove_single_variable_constraints"); + d.name("tighten_bounds"); + e.name("remove_fixed_variables"); + omega.name("conditional"); + + alpha.precede(a); + a.precede(b); + b.precede(c); + c.precede(d); + d.precede(e); + e.precede(omega); + omega.precede(a); // loops back to (a) iff omega returns 0; o/w this will exit the taskflow +} + +template +void Presolver::load_taskflow_cleanup() { + auto a = tf_helper_.taskflow_cleanup.emplace( + [&]() { technique_remove_invalid_markers(); } + ); + a.name("remove_invalid_markers"); } template diff --git a/extern/taskflow b/extern/taskflow new file mode 160000 index 0000000..6633a09 --- /dev/null +++ b/extern/taskflow @@ -0,0 +1 @@ +Subproject commit 6633a0919065f27b6697a8fb51ed36fbda8d384a diff --git a/setup.py b/setup.py index 4ea96c4..1daf620 100644 --- a/setup.py +++ b/setup.py @@ -26,6 +26,7 @@ 'msvc': ['/std:c++17', '/EHsc'], 'unix': [ '-std=c++17', + '-pthread' ], } @@ -62,6 +63,7 @@ def build_extensions(self): include_dirs=[ numpy.get_include(), dimod.get_include(), + 'extern/taskflow' ], install_requires=[ 'numpy>=1.20.0,<2.0.0', # keep synced with circle-ci, pyproject.toml diff --git a/testscpp/Makefile b/testscpp/Makefile index a565fe2..0c4a184 100644 --- a/testscpp/Makefile +++ b/testscpp/Makefile @@ -1,11 +1,12 @@ -ROOT := ../ +ROOT := .. SRC := $(ROOT)/dwave/preprocessing/ CATCH2 := $(ROOT)/testscpp/Catch2/single_include/ DIMOD := $(shell python -c 'import dimod; print(dimod.get_include())') -INCLUDES := -I $(SRC)/include/ -I $(DIMOD) -I $(CATCH2) +TASKFLOW := $(ROOT)/extern/taskflow/ +INCLUDES := -I $(SRC)/include/ -I $(DIMOD) -I $(CATCH2) -I $(TASKFLOW) FLAGS := -std=c++17 -Wall -Wno-unknown-pragmas -Wno-sign-compare -Wno-deprecated-declarations -fcompare-debug-second -O3 -all: catch2 test_main test_main_parallel tests tests_parallel +all: update test_main test_main_parallel tests tests_parallel tests: test_main.out ./test_main @@ -14,13 +15,14 @@ tests_parallel: test_main_parallel.out ./test_main_parallel test_main: test_main.cpp - g++ $(FLAGS) -c test_main.cpp - g++ $(FLAGS) test_main.o tests/*.cpp -o test_main $(INCLUDES) + g++ $(FLAGS) -pthread -c test_main.cpp + g++ $(FLAGS) -pthread test_main.o tests/*.cpp -o test_main $(INCLUDES) + test_main_parallel: test_main.cpp g++ $(FLAGS) -fopenmp -c test_main.cpp -o test_main_parallel.o g++ $(FLAGS) -fopenmp test_main_parallel.o tests/*.cpp -o test_main_parallel $(INCLUDES) -catch2: +update: git submodule init git submodule update From 4b3584bdafeaa88e252b33b976bdffba1bbcab81 Mon Sep 17 00:00:00 2001 From: mcoury Date: Thu, 15 Dec 2022 12:34:35 -0800 Subject: [PATCH 2/3] Parallelizing some techniques using std::for_each; note this requires a corresponsing change in dimod --- .../preprocessing/include/dwave/presolve.hpp | 119 ++++++++---------- 1 file changed, 54 insertions(+), 65 deletions(-) diff --git a/dwave/preprocessing/include/dwave/presolve.hpp b/dwave/preprocessing/include/dwave/presolve.hpp index e0bc4ab..fe8d991 100644 --- a/dwave/preprocessing/include/dwave/presolve.hpp +++ b/dwave/preprocessing/include/dwave/presolve.hpp @@ -15,13 +15,14 @@ #pragma once #include +#include #include #include #include +#include "dimod/constrained_quadratic_model.h" #include "taskflow/core/taskflow.hpp" #include "taskflow/taskflow.hpp" -#include "dimod/constrained_quadratic_model.h" namespace dwave { namespace presolve { @@ -159,13 +160,11 @@ class Presolver { bool loop_changed; bool model_feasible = true; - bool operator=(const struct TfHelper& that) { - return true; - } + bool operator=(const struct TfHelper& that) { return true; } }; struct TfHelper tf_helper_; - + void load_taskflow_onetime(); void load_taskflow_trivial(int max_rounds = 100); void load_taskflow_cleanup(); @@ -212,21 +211,24 @@ class Presolver { } } void technique_remove_offsets() { - for (size_type c = 0; c < model_.num_constraints(); ++c) { - auto& constraint = model_.constraint_ref(c); - if (constraint.offset()) { - constraint.set_rhs(constraint.rhs() - constraint.offset()); - constraint.set_offset(0); - } - } + auto constraints = model_.get_constraints(); + std::for_each( + std::execution::par_unseq, constraints.begin(), constraints.end(), + [](auto&& constraint_ptr) { + if (constraint_ptr->offset()) { + constraint_ptr->set_rhs(constraint_ptr->rhs() - constraint_ptr->offset()); + constraint_ptr->set_offset(0); + } + }); } void technique_flip_constraints() { - for (size_type c = 0; c < model_.num_constraints(); ++c) { - auto& constraint = model_.constraint_ref(c); - if (constraint.sense() == dimod::Sense::GE) { - constraint.scale(-1); - } - } + auto constraints = model_.get_constraints(); + std::for_each(std::execution::par_unseq, constraints.begin(), constraints.end(), + [](auto&& constraint_ptr) { + if (constraint_ptr->sense() == dimod::Sense::GE) { + constraint_ptr->scale(-1); + } + }); } void technique_remove_self_loops() { std::unordered_map mapping; @@ -359,9 +361,11 @@ class Presolver { bool ret = false; ret |= remove_zero_biases(model_.objective); - for (size_t c = 0; c < model_.num_constraints(); ++c) { - ret |= remove_zero_biases(model_.constraint_ref(c)); - } + auto constraints = model_.get_constraints(); + std::for_each(std::execution::par_unseq, constraints.begin(), constraints.end(), + [&](auto&& constraint_ptr) { + ret |= remove_zero_biases(*constraint_ptr); + }); return ret; } @@ -392,7 +396,7 @@ class Presolver { return ret; } bool technique_remove_fixed_variables() { - bool ret = false; + bool ret = false; size_type v = 0; while (v < model_.num_variables()) { if (model_.lower_bound(v) == model_.upper_bound(v)) { @@ -446,10 +450,9 @@ void Presolver::apply() { tf_helper_.executor.run(tf_helper_.taskflow_onetime).wait(); tf_helper_.executor.run(tf_helper_.taskflow_trivial).wait(); - if(tf_helper_.model_feasible) { + if (tf_helper_.model_feasible) { tf_helper_.executor.run(tf_helper_.taskflow_cleanup).wait(); - } - else { + } else { // need this exact message for Python throw std::logic_error("infeasible"); } @@ -478,11 +481,8 @@ void Presolver::load_default_presolvers( template void Presolver::load_taskflow_onetime() { auto [a, b, c, d] = tf_helper_.taskflow_onetime.emplace( - [&]() { technique_spin_to_binary(); }, - [&]() { technique_remove_offsets(); }, - [&]() { technique_flip_constraints(); }, - [&]() { technique_remove_self_loops(); } - ); + [&]() { technique_spin_to_binary(); }, [&]() { technique_remove_offsets(); }, + [&]() { technique_flip_constraints(); }, [&]() { technique_remove_self_loops(); }); a.name("spin_to_binary"); b.name("remove_offsets"); @@ -496,42 +496,33 @@ void Presolver::load_taskflow_onetime() template void Presolver::load_taskflow_trivial(int max_rounds) { - auto alpha = tf_helper_.taskflow_trivial.emplace( - [&]() { - tf_helper_.loop_changed = false; - tf_helper_.loop_counter = 0; - } - ); + auto alpha = tf_helper_.taskflow_trivial.emplace([&]() { + tf_helper_.loop_changed = false; + tf_helper_.loop_counter = 0; + }); auto [a, b, c, d, e] = tf_helper_.taskflow_trivial.emplace( - [&]() { tf_helper_.loop_changed |= technique_remove_zero_biases(); }, - [&]() { tf_helper_.loop_changed |= technique_check_for_nan(); }, - [&]() { tf_helper_.loop_changed |= technique_remove_single_variable_constraints(); }, - [&]() - { - if(tf_helper_.model_feasible) { - tf_helper_.loop_changed |= technique_tighten_bounds(); + [&]() { tf_helper_.loop_changed |= technique_remove_zero_biases(); }, + [&]() { tf_helper_.loop_changed |= technique_check_for_nan(); }, + [&]() { tf_helper_.loop_changed |= technique_remove_single_variable_constraints(); }, + [&]() { + if (tf_helper_.model_feasible) { + tf_helper_.loop_changed |= technique_tighten_bounds(); } }, - [&]() - { - if(tf_helper_.model_feasible) { - tf_helper_.loop_changed |= technique_remove_fixed_variables(); + [&]() { + if (tf_helper_.model_feasible) { + tf_helper_.loop_changed |= technique_remove_fixed_variables(); } - } - ); - auto omega = tf_helper_.taskflow_trivial.emplace( - [&]() { - if(tf_helper_.model_feasible - && tf_helper_.loop_changed - && ++tf_helper_.loop_counter < max_rounds - ) { - tf_helper_.loop_changed = false; - return 0; // This will take us back to (a) - } - return 1; // This will cause us to exit + }); + auto omega = tf_helper_.taskflow_trivial.emplace([&]() { + if (tf_helper_.model_feasible && tf_helper_.loop_changed && + ++tf_helper_.loop_counter < max_rounds) { + tf_helper_.loop_changed = false; + return 0; // This will take us back to (a) } - ); - + return 1; // This will cause us to exit + }); + alpha.name("initialize"); a.name("remove_zero_biases"); b.name("check_for_nan"); @@ -546,14 +537,12 @@ void Presolver::load_taskflow_trivial(in c.precede(d); d.precede(e); e.precede(omega); - omega.precede(a); // loops back to (a) iff omega returns 0; o/w this will exit the taskflow + omega.precede(a); // loops back to (a) iff omega returns 0; o/w this will exit the taskflow } template void Presolver::load_taskflow_cleanup() { - auto a = tf_helper_.taskflow_cleanup.emplace( - [&]() { technique_remove_invalid_markers(); } - ); + auto a = tf_helper_.taskflow_cleanup.emplace([&]() { technique_remove_invalid_markers(); }); a.name("remove_invalid_markers"); } From 43654064cdaa7ebf4b15f4806b808bb1ee6eeaef Mon Sep 17 00:00:00 2001 From: mcoury Date: Mon, 19 Dec 2022 13:46:11 -0800 Subject: [PATCH 3/3] Updated to use changes on arcondello/dimod/tree/feature/constraint-iterator --- dwave/preprocessing/include/dwave/presolve.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dwave/preprocessing/include/dwave/presolve.h b/dwave/preprocessing/include/dwave/presolve.h index a71c480..585dea5 100644 --- a/dwave/preprocessing/include/dwave/presolve.h +++ b/dwave/preprocessing/include/dwave/presolve.h @@ -212,7 +212,7 @@ class Presolver { } } void technique_remove_offsets() { - auto constraints = model_.get_constraints(); + auto constraints = model_.constraints; std::for_each( std::execution::par_unseq, constraints.begin(), constraints.end(), [](auto&& constraint_ptr) { @@ -223,7 +223,7 @@ class Presolver { }); } void technique_flip_constraints() { - auto constraints = model_.get_constraints(); + auto constraints = model_.constraints; std::for_each(std::execution::par_unseq, constraints.begin(), constraints.end(), [](auto&& constraint_ptr) { if (constraint_ptr->sense() == dimod::Sense::GE) { @@ -362,7 +362,7 @@ class Presolver { bool ret = false; ret |= remove_zero_biases(model_.objective); - auto constraints = model_.get_constraints(); + auto constraints = model_.constraints; std::for_each(std::execution::par_unseq, constraints.begin(), constraints.end(), [&](auto&& constraint_ptr) { ret |= remove_zero_biases(*constraint_ptr);