From 0aa8af7993ab5c9a39af4ea72876ebd264a3cab5 Mon Sep 17 00:00:00 2001 From: Lee Spector Date: Wed, 30 Aug 2023 12:02:46 -0400 Subject: [PATCH 1/5] Add pmapallv utility function --- src/propeller/utils.cljc | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/propeller/utils.cljc b/src/propeller/utils.cljc index 5035f02..aa0d7ba 100755 --- a/src/propeller/utils.cljc +++ b/src/propeller/utils.cljc @@ -85,3 +85,22 @@ zip/path count (max height)))))) + +(defn pmapallv + "A utility for concurrent execution of a function on items in a collection. +In single-thread-mode this acts like mapv. Otherwise it acts like pmap but: +1) coll should be finite, 2) the returned sequence will not be lazy, and will +in fact be a vector, 3) calls to f may occur in any order, to maximize +multicore processor utilization, and 4) takes only one coll so far." + [f coll args] + #?(:clj (vec (if (:single-thread-mode args) + (doall (map f coll)) + (let [agents (map #(agent % :error-handler + (fn [agnt except] + (clojure.repl/pst except 1000) + (System/exit 0))) + coll)] + (dorun (map #(send % f) agents)) + (apply await agents) + (doall (map deref agents))))) + :cljs (mapv f coll))) From 4faf2300ee43a731211afe1b0b9d7b04a1ef4124 Mon Sep 17 00:00:00 2001 From: Lee Spector Date: Wed, 30 Aug 2023 12:14:24 -0400 Subject: [PATCH 2/5] Remove most of GP function's docstring --- src/propeller/gp.cljc | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/src/propeller/gp.cljc b/src/propeller/gp.cljc index f7a3a97..84ec5ac 100644 --- a/src/propeller/gp.cljc +++ b/src/propeller/gp.cljc @@ -32,25 +32,7 @@ (println))) (defn gp - "Main GP loop. - -On each iteration, it creates a population of random plushies using a mapper -function and genome/make-random-plushy function, -then it sorts the population by the total error using the error-function -and sort-by function. It then takes the best individual from the sorted population, -and if the parent selection is set to epsilon-lexicase, it adds the epsilons to the argmap. - -The function then checks if the custom-report argument is set, -if so it calls that function passing the evaluated population, -current generation and argmap. If not, it calls the report function -passing the evaluated population, current generation and argmap. - -Then, it checks if the total error of the best individual is less than or equal -to the solution-error-threshold or if the current generation is greater than or -equal to the max-generations specified. If either is true, the function -exits with the best individual or nil. If not, it creates new individuals -for the next generation using the variation/new-individual function and the -repeatedly function, and then continues to the next iteration of the loop. " + "Main GP loop." [{:keys [population-size max-generations error-function instructions max-initial-plushy-size solution-error-threshold mapper] :or {solution-error-threshold 0.0 From a2192ca37d047e31305a9c7f53bd0df74d1c5ebd Mon Sep 17 00:00:00 2001 From: Lee Spector Date: Wed, 30 Aug 2023 12:36:47 -0400 Subject: [PATCH 3/5] Use pmapallv for evaluation in GP, after debugging and adding shutdown-agents --- src/propeller/gp.cljc | 36 +++++++++++++++++++++--------------- src/propeller/utils.cljc | 5 +++-- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/src/propeller/gp.cljc b/src/propeller/gp.cljc index 84ec5ac..32c23af 100644 --- a/src/propeller/gp.cljc +++ b/src/propeller/gp.cljc @@ -13,7 +13,8 @@ [propeller.push.instructions.polymorphic] [propeller.push.instructions.string] [propeller.push.instructions.vector] - [propeller.selection :as selection])) + [propeller.selection :as selection] + [propeller.utils :as utils])) (defn report "Reports information for each generation." @@ -34,24 +35,23 @@ (defn gp "Main GP loop." [{:keys [population-size max-generations error-function instructions - max-initial-plushy-size solution-error-threshold mapper] - :or {solution-error-threshold 0.0 - ;; The `mapper` will perform a `map`-like operation to apply a function to every individual - ;; in the population. The default is `map` but other options include `mapv`, or `pmap`. - mapper #?(:clj pmap :cljs map)} + max-initial-plushy-size solution-error-threshold] + :or {solution-error-threshold 0.0} :as argmap}] ;; (prn {:starting-args (update (update argmap :error-function str) :instructions str)}) (println) ;; (loop [generation 0 - population (mapper - (fn [_] {:plushy (genome/make-random-plushy instructions max-initial-plushy-size)}) - (range population-size))] ;creates population of random plushys + population (utils/pmapallv + (fn [_] {:plushy (genome/make-random-plushy instructions max-initial-plushy-size)}) + (range population-size) + argmap)] ;creates population of random plushys (let [evaluated-pop (sort-by :total-error - (mapper - (partial error-function argmap (:training-data argmap)) - population)) ;population sorted by :total-error + (utils/pmapallv + (partial error-function argmap (:training-data argmap)) + population ;population sorted by :total-error + argmap)) best-individual (first evaluated-pop) argmap (if (= (:parent-selection argmap) :epsilon-lexicase) (assoc argmap :epsilons (selection/epsilon-list evaluated-pop)) @@ -66,11 +66,17 @@ (prn {:total-test-error (:total-error (error-function argmap (:testing-data argmap) best-individual))}) (when (:simplification? argmap) - (let [simplified-plushy (simplification/auto-simplify-plushy (:plushy best-individual) error-function argmap)] - (prn {:total-test-error-simplified (:total-error (error-function argmap (:testing-data argmap) (hash-map :plushy simplified-plushy)))})))) + (let [simplified-plushy (simplification/auto-simplify-plushy + (:plushy best-individual) + error-function argmap)] + (prn {:total-test-error-simplified + (:total-error (error-function argmap + (:testing-data argmap) + (hash-map :plushy simplified-plushy)))}))) + #?(:clj (shutdown-agents))) ;; (>= generation max-generations) - nil + #?(:clj (shutdown-agents)) ;; :else (recur (inc generation) (if (:elitism argmap) diff --git a/src/propeller/utils.cljc b/src/propeller/utils.cljc index aa0d7ba..4ac1cc8 100755 --- a/src/propeller/utils.cljc +++ b/src/propeller/utils.cljc @@ -1,6 +1,7 @@ (ns propeller.utils "Useful functions." - (:require [clojure.zip :as zip])) + (:require [clojure.zip :as zip] + [clojure.repl :as repl])) (defn first-non-nil "Returns the first non-nil values from the collection, or returns `nil` if @@ -97,7 +98,7 @@ multicore processor utilization, and 4) takes only one coll so far." (doall (map f coll)) (let [agents (map #(agent % :error-handler (fn [agnt except] - (clojure.repl/pst except 1000) + (repl/pst except 1000) (System/exit 0))) coll)] (dorun (map #(send % f) agents)) From 87ba635f4f654b4533c9c79b874d812e426d97de Mon Sep 17 00:00:00 2001 From: Lee Spector Date: Wed, 30 Aug 2023 12:52:34 -0400 Subject: [PATCH 4/5] Use pmapallv for variation, which includes selection --- src/propeller/gp.cljc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/propeller/gp.cljc b/src/propeller/gp.cljc index 32c23af..f7d3ff9 100644 --- a/src/propeller/gp.cljc +++ b/src/propeller/gp.cljc @@ -80,9 +80,11 @@ ;; :else (recur (inc generation) (if (:elitism argmap) - (conj (repeatedly (dec population-size) - #(variation/new-individual evaluated-pop argmap)) + (conj (utils/pmapallv (fn [_] (variation/new-individual evaluated-pop argmap)) + (range (dec population-size)) + argmap) (first evaluated-pop)) ;elitism maintains the most-fit individual - (repeatedly population-size - #(variation/new-individual evaluated-pop argmap)))))))) + (utils/pmapallv (fn [_] (variation/new-individual evaluated-pop argmap)) + (range population-size) + argmap))))))) From 7797fc1def7d6f61d9a51a482550055b2fd5142e Mon Sep 17 00:00:00 2001 From: Lee Spector Date: Wed, 30 Aug 2023 13:14:32 -0400 Subject: [PATCH 5/5] Document concurrency and :single-thread-mode in README --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 0e18818..f3a17c1 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,12 @@ or lein run -m propeller.problems.simple-regression :variation "{:umad 1.0}" ``` +By default, Propeller will conduct many processes concurrently on multiple +cores using threads. If you want to disable this behavior (for example, during +debugging) then provide the argument `:single-thread-mode` with the value `true`. +Threads are not available in Javascript, so no processes are run concurrnetly +when Propeller is run in Clojurescript. + ## CLJS Usage