From a215c1c271a981afa5c174176fbaffa300b2cb53 Mon Sep 17 00:00:00 2001 From: Ryan Boldi Date: Sat, 26 Feb 2022 21:04:29 -0500 Subject: [PATCH 1/6] added more communicative autosimplification tests --- test/propeller/utils_test.cljc | 46 ++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/test/propeller/utils_test.cljc b/test/propeller/utils_test.cljc index d11ea87..15a4984 100644 --- a/test/propeller/utils_test.cljc +++ b/test/propeller/utils_test.cljc @@ -32,9 +32,55 @@ (t/is (= 1 (u/count-points '()))) (t/is (= 2 (u/count-points '(:a))))) +<<<<<<< Updated upstream ;(t/deftest seq-zip-test ; (t/is )) ;(t/deftest depth-test ; (t/is (= 3 (u/depth ())))) +======= +(t/testing "choose-random-k" + (t/testing "should return indices that are a member of the original array" + (t/is (every? identity (map #(contains? (set (range 10)) %) (s/choose-random-k 3 (range 10)))))) + (t/testing "should return a list of size k" + (t/is (= (count (s/choose-random-k 7 (range 10))) 7)))) + +(t/testing "delete-at-indices" + (t/testing "should actually remove indicated items" + (t/is (= '(:hi1 :hi2) (s/delete-at-indices '(0 3) '(:hi0 :hi1 :hi2 :hi3))))) + (t/testing "should work with numerical indices" + (t/is (= '(:hi1 :hi2 :hi3) (s/delete-at-indices '(0) '(:hi0 :hi1 :hi2 :hi3))))) + (t/testing "should not delete anything for index out of bounds" + (t/is (= '(:hi1 :hi2 :hi3) (s/delete-at-indices '(0 10) '(:hi0 :hi1 :hi2 :hi3)))) + (t/is (= '(:hi1 :hi2 :hi3) (s/delete-at-indices '(0 -10) '(:hi0 :hi1 :hi2 :hi3)))) + (t/is (= '(:hi1 :hi2 :hi3) (s/delete-at-indices '(-0 -10) '(:hi0 :hi1 :hi2 :hi3))))) + (t/testing "should only delete at single index once" + (t/is (= '(:hi1 :hi2) (s/delete-at-indices '(0 0 0 0 3 3 3) '(:hi0 :hi1 :hi2 :hi3))))) + (t/testing "should return empty list when deleting from empty list" + (t/is (= '() (s/delete-at-indices '(0) '())))) + (t/testing "should be able to delete at arbitrary indices" + (t/is (= (count (s/delete-at-indices (s/choose-random-k 3 (range 10)) (range 10))) 7)))) + +(t/testing "delete-random-k" + (t/testing "should remove the correct amount of items" + (t/is (= (count (s/delete-k-random 3 (range 10))) 7)) + (t/is (= (count (s/delete-k-random 10 (range 10))) 0)) + (t/is (= (count (s/delete-k-random 0 (range 10))) 10))) + (t/testing "should not fail if k >> size of collection" + (t/is (= (count (s/delete-k-random 300 (range 10))) 0)) + (t/is (= (s/delete-k-random 300 '(:hi1 :hi2 :hi3)) '()))) + (t/testing "should not fail if the collection is empty" + (t/is (= (count (s/delete-k-random 300 '())) 0)) + (t/is (= (count (s/delete-k-random 0 '())) 0))) + (t/testing "should maintain order of the remaining items" + (t/is (apply < (s/delete-k-random 3 (range 10)))))) + +(t/testing "auto-simplify-plushy" + (t/testing "should handle having an empty plushy" + (t/is (= (s/auto-simplify-plushy {} '() 100 (fn [argmap data plushy] 0) {} 3 false) '()))) + (let [plushy '(:exec_dup 1 :integer_add close :in1 :integer_add 0 :in1 :in1 :integer_mult :integer_add)] + (t/testing "should decrease size of plushy that always has perfect scores" + (t/is (< (count (s/auto-simplify-plushy {} plushy 5 (fn [argmap data plushy] 0) {} 3 false)) (count plushy))) + (t/is (< (count (s/auto-simplify-plushy {} plushy 1 (fn [argmap data plushy] 0) {} 10 false)) (count plushy)))))) +>>>>>>> Stashed changes From e25dab4798eafc7b59b10213b360c9c48dd99566 Mon Sep 17 00:00:00 2001 From: Ryan Boldi Date: Thu, 13 Jan 2022 17:26:46 -0500 Subject: [PATCH 2/6] Implements auto-simplification using k-annealing & tests Select k random elements in plushy. Delete these, and see if the plushy's behaviors are still identical. If they are, this is now the current plushy. If not, decrease k with a certain prob, and try again. --- src/propeller/gp.cljc | 6 ++++- src/propeller/simplification.cljc | 37 +++++++++++++++++++++++++++++++ test/propeller/utils_test.cljc | 16 ++++--------- 3 files changed, 46 insertions(+), 13 deletions(-) mode change 100755 => 100644 src/propeller/gp.cljc create mode 100644 src/propeller/simplification.cljc diff --git a/src/propeller/gp.cljc b/src/propeller/gp.cljc old mode 100755 new mode 100644 index 0d05a83..7c87229 --- a/src/propeller/gp.cljc +++ b/src/propeller/gp.cljc @@ -2,6 +2,7 @@ (:require [clojure.string] [clojure.pprint] [propeller.genome :as genome] + [propeller.simplification :as simplification] [propeller.variation :as variation] [propeller.push.instructions.bool] [propeller.push.instructions.character] @@ -58,7 +59,10 @@ (<= (:total-error best-individual) solution-error-threshold) (do (prn {:success-generation generation}) (prn {:total-test-error - (:total-error (error-function argmap (:testing-data argmap) best-individual))})) + (:total-error (error-function argmap (:testing-data argmap) best-individual))}) + (if (:simplification? argmap) + (let [simplified-plushy (simplification/auto-simplify-plushy argmap (:plushy best-individual) (:simplification-steps argmap) error-function (:training-data argmap) (:simplification-k argmap) (:simplification-k-prob argmap) (:simplification-verbose? argmap))] + (prn {:total-test-error-simplified (:total-error (error-function argmap (:testing-data argmap) (hash-map :plushy simplified-plushy)))})))) ;; (>= generation max-generations) nil diff --git a/src/propeller/simplification.cljc b/src/propeller/simplification.cljc new file mode 100644 index 0000000..5527be0 --- /dev/null +++ b/src/propeller/simplification.cljc @@ -0,0 +1,37 @@ +(ns propeller.simplification + (:require [propeller.genome :as genome] + [propeller.push.interpreter :as interpreter] + [propeller.push.state :as state] + [propeller.tools.math :as math] + )) + +(defn choose-random-k + [k indices] + (take k (shuffle indices))) + +(defn delete-at-indices + "deletes the values at given set of indices" + [indices plushy] + (let [sorted-indices (sort > indices)] + (keep-indexed #(if (not (some #{%1} sorted-indices)) %2) plushy))) + +(defn delete-k-random + [k plushy] + (delete-at-indices (choose-random-k k (range (count plushy))) plushy)) + +(defn auto-simplify-plushy + "naive auto-simplification with simple k annealing" + [argmap plushy steps error-function training-data start-k decrease-k-on-failure-prob verbose?] + (if verbose? (prn "Starting Auto-Simplification" {:current-plushy-length (count plushy) :k start-k :decrease-k-on-failure-prob decrease-k-on-failure-prob})) + (let [initial-errors (:errors (error-function argmap training-data {:plushy plushy}))] + (loop [step 0 curr-plushy plushy k start-k] + (if (and verbose? (= (mod step 50) 0)) (pr {:step step :k k} " ")) + (if (< steps step) + (do (if verbose? (prn "Finished Auto-Simplification" {:final-plushy-length (count curr-plushy) :final-plushy curr-plushy})) curr-plushy) + (let [new-plushy (delete-k-random k curr-plushy) + new-plushy-errors (:errors (error-function argmap training-data {:plushy new-plushy})) + new-equal? (= new-plushy-errors initial-errors)] + (print-str new-plushy new-plushy-errors new-equal?) + (recur (inc step) + (if new-equal? new-plushy curr-plushy) + (if new-equal? k (if (and (> k 1) (< (rand) decrease-k-on-failure-prob)) (dec k) k)))))))) \ No newline at end of file diff --git a/test/propeller/utils_test.cljc b/test/propeller/utils_test.cljc index 15a4984..e450a38 100644 --- a/test/propeller/utils_test.cljc +++ b/test/propeller/utils_test.cljc @@ -1,6 +1,8 @@ (ns propeller.utils-test (:require [clojure.test :as t] - [propeller.utils :as u])) + [propeller.utils :as u] + [propeller.simplification :as s])) + (t/deftest first-non-nil-test (t/is (= 1 (u/first-non-nil '(1 2 3)))) (t/is (= nil (u/first-non-nil []))) @@ -26,20 +28,11 @@ true (= 2 test)))))) - (t/deftest count-points-test (t/is (= 6 (u/count-points '(:a :b (:c :d))))) (t/is (= 1 (u/count-points '()))) (t/is (= 2 (u/count-points '(:a))))) -<<<<<<< Updated upstream -;(t/deftest seq-zip-test -; (t/is )) - -;(t/deftest depth-test -; (t/is (= 3 (u/depth ())))) - -======= (t/testing "choose-random-k" (t/testing "should return indices that are a member of the original array" (t/is (every? identity (map #(contains? (set (range 10)) %) (s/choose-random-k 3 (range 10)))))) @@ -82,5 +75,4 @@ (let [plushy '(:exec_dup 1 :integer_add close :in1 :integer_add 0 :in1 :in1 :integer_mult :integer_add)] (t/testing "should decrease size of plushy that always has perfect scores" (t/is (< (count (s/auto-simplify-plushy {} plushy 5 (fn [argmap data plushy] 0) {} 3 false)) (count plushy))) - (t/is (< (count (s/auto-simplify-plushy {} plushy 1 (fn [argmap data plushy] 0) {} 10 false)) (count plushy)))))) ->>>>>>> Stashed changes + (t/is (< (count (s/auto-simplify-plushy {} plushy 1 (fn [argmap data plushy] 0) {} 10 false)) (count plushy)))))) \ No newline at end of file From f369d23ad554c5f895a2da2a3ded39965b0847e5 Mon Sep 17 00:00:00 2001 From: Ryan Boldi Date: Thu, 13 Jan 2022 20:31:17 -0500 Subject: [PATCH 3/6] simplified simplification.cljc --- src/propeller/simplification.cljc | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/propeller/simplification.cljc b/src/propeller/simplification.cljc index 5527be0..fbd47b1 100644 --- a/src/propeller/simplification.cljc +++ b/src/propeller/simplification.cljc @@ -20,18 +20,15 @@ (delete-at-indices (choose-random-k k (range (count plushy))) plushy)) (defn auto-simplify-plushy - "naive auto-simplification with simple k annealing" - [argmap plushy steps error-function training-data start-k decrease-k-on-failure-prob verbose?] - (if verbose? (prn "Starting Auto-Simplification" {:current-plushy-length (count plushy) :k start-k :decrease-k-on-failure-prob decrease-k-on-failure-prob})) + "naive auto-simplification" + [argmap plushy steps error-function training-data k verbose?] + (if verbose? (prn {:start-plushy-length (count plushy) :k k})) (let [initial-errors (:errors (error-function argmap training-data {:plushy plushy}))] - (loop [step 0 curr-plushy plushy k start-k] - (if (and verbose? (= (mod step 50) 0)) (pr {:step step :k k} " ")) + (loop [step 0 curr-plushy plushy] (if (< steps step) - (do (if verbose? (prn "Finished Auto-Simplification" {:final-plushy-length (count curr-plushy) :final-plushy curr-plushy})) curr-plushy) - (let [new-plushy (delete-k-random k curr-plushy) + (do (if verbose? (prn {:final-plushy-length (count curr-plushy) :final-plushy curr-plushy})) curr-plushy) + (let [new-plushy (delete-k-random (rand-int k) curr-plushy) new-plushy-errors (:errors (error-function argmap training-data {:plushy new-plushy})) new-equal? (= new-plushy-errors initial-errors)] - (print-str new-plushy new-plushy-errors new-equal?) (recur (inc step) - (if new-equal? new-plushy curr-plushy) - (if new-equal? k (if (and (> k 1) (< (rand) decrease-k-on-failure-prob)) (dec k) k)))))))) \ No newline at end of file + (if new-equal? new-plushy curr-plushy))))))) \ No newline at end of file From af22f9793b70339b9c90cfa2b99c327426840471 Mon Sep 17 00:00:00 2001 From: Ryan Boldi Date: Sat, 19 Feb 2022 16:52:11 -0500 Subject: [PATCH 4/6] autosimplification implementation complete --- .gitignore | 1 + src/propeller/gp.cljc | 2 +- src/propeller/problems/PSB2/fuel_cost.cljc | 14 ++++++++ src/propeller/problems/simple_regression.cljc | 35 +++++++++++-------- 4 files changed, 37 insertions(+), 15 deletions(-) mode change 100755 => 100644 src/propeller/problems/simple_regression.cljc diff --git a/.gitignore b/.gitignore index df3fb06..0fc1327 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,4 @@ node_modules/ # https://github.com/thelmuth/program-synthesis-benchmark-datasets /data **/.DS_Store +.shadow-cljs/classpath.edn diff --git a/src/propeller/gp.cljc b/src/propeller/gp.cljc index 7c87229..3c89f00 100644 --- a/src/propeller/gp.cljc +++ b/src/propeller/gp.cljc @@ -61,7 +61,7 @@ (prn {:total-test-error (:total-error (error-function argmap (:testing-data argmap) best-individual))}) (if (:simplification? argmap) - (let [simplified-plushy (simplification/auto-simplify-plushy argmap (:plushy best-individual) (:simplification-steps argmap) error-function (:training-data argmap) (:simplification-k argmap) (:simplification-k-prob argmap) (:simplification-verbose? argmap))] + (let [simplified-plushy (simplification/auto-simplify-plushy argmap (:plushy best-individual) (:simplification-steps argmap) error-function (:training-data argmap) (:simplification-k argmap) (:simplification-verbose? argmap))] (prn {:total-test-error-simplified (:total-error (error-function argmap (:testing-data argmap) (hash-map :plushy simplified-plushy)))})))) ;; (>= generation max-generations) diff --git a/src/propeller/problems/PSB2/fuel_cost.cljc b/src/propeller/problems/PSB2/fuel_cost.cljc index 9d45455..9462ebe 100644 --- a/src/propeller/problems/PSB2/fuel_cost.cljc +++ b/src/propeller/problems/PSB2/fuel_cost.cljc @@ -1,6 +1,7 @@ (ns propeller.problems.PSB2.fuel-cost (:require [psb2.core :as psb2] [propeller.genome :as genome] + [propeller.simplification :as simplification] [propeller.push.interpreter :as interpreter] [propeller.utils :as utils] [propeller.push.instructions :refer [get-stack-instructions]] @@ -81,3 +82,16 @@ :elitism false} (apply hash-map (map #(if (string? %) (read-string %) %) args)))) (#?(:clj shutdown-agents))) + + +(defn fuel-cost-autosimplify + [plushy] + (simplification/auto-simplify-plushy {:instructions instructions + :error-function error-function + :training-data (:train train-and-test-data) + :testing-data (:test train-and-test-data) + :max-generations 300 + :population-size 1000 + :max-initial-plushy-size 250 + :step-limit 2000} plushy 5000 error-function (:train train-and-test-data) 5 true)) + diff --git a/src/propeller/problems/simple_regression.cljc b/src/propeller/problems/simple_regression.cljc old mode 100755 new mode 100644 index d0f153e..dd0a063 --- a/src/propeller/problems/simple_regression.cljc +++ b/src/propeller/problems/simple_regression.cljc @@ -66,18 +66,25 @@ [& args] (gp/gp (merge - {:instructions instructions - :error-function error-function - :training-data (:train train-and-test-data) - :testing-data (:test train-and-test-data) - :max-generations 500 - :population-size 500 - :max-initial-plushy-size 100 - :step-limit 200 - :parent-selection :lexicase - :tournament-size 5 - :umad-rate 0.1 - :variation {:umad 0.5 :crossover 0.5} - :elitism false} - (apply hash-map (map #(if (string? %) (read-string %) %) args)))) + {:instructions instructions + :error-function error-function + :training-data (:train train-and-test-data) + :testing-data (:test train-and-test-data) + :max-generations 500 + :population-size 500 + :max-initial-plushy-size 100 + :step-limit 200 + :parent-selection :lexicase + :case-queue? false + :case-step 1 + :downsample-size 10 + :tournament-size 5 + :umad-rate 0.1 + :variation {:umad 0.5 :crossover 0.5} + :elitism false + :simplification? true + :simplification-steps 100 + :simplification-k 5 + :simplification-verbose? true} + (apply hash-map (map #(if (string? %) (read-string %) %) args)))) (#?(:clj shutdown-agents))) From f1838b6b32e60a0094cbcc9738f84cf64553e7fb Mon Sep 17 00:00:00 2001 From: Ryan Boldi Date: Sat, 26 Feb 2022 21:11:38 -0500 Subject: [PATCH 5/6] autosimplification implementation complete --- src/propeller/gp.cljc | 2 +- src/propeller/problems/simple_regression.cljc | 35 ++++++++----------- 2 files changed, 15 insertions(+), 22 deletions(-) diff --git a/src/propeller/gp.cljc b/src/propeller/gp.cljc index 3c89f00..7c87229 100644 --- a/src/propeller/gp.cljc +++ b/src/propeller/gp.cljc @@ -61,7 +61,7 @@ (prn {:total-test-error (:total-error (error-function argmap (:testing-data argmap) best-individual))}) (if (:simplification? argmap) - (let [simplified-plushy (simplification/auto-simplify-plushy argmap (:plushy best-individual) (:simplification-steps argmap) error-function (:training-data argmap) (:simplification-k argmap) (:simplification-verbose? argmap))] + (let [simplified-plushy (simplification/auto-simplify-plushy argmap (:plushy best-individual) (:simplification-steps argmap) error-function (:training-data argmap) (:simplification-k argmap) (:simplification-k-prob argmap) (:simplification-verbose? argmap))] (prn {:total-test-error-simplified (:total-error (error-function argmap (:testing-data argmap) (hash-map :plushy simplified-plushy)))})))) ;; (>= generation max-generations) diff --git a/src/propeller/problems/simple_regression.cljc b/src/propeller/problems/simple_regression.cljc index dd0a063..d0f153e 100644 --- a/src/propeller/problems/simple_regression.cljc +++ b/src/propeller/problems/simple_regression.cljc @@ -66,25 +66,18 @@ [& args] (gp/gp (merge - {:instructions instructions - :error-function error-function - :training-data (:train train-and-test-data) - :testing-data (:test train-and-test-data) - :max-generations 500 - :population-size 500 - :max-initial-plushy-size 100 - :step-limit 200 - :parent-selection :lexicase - :case-queue? false - :case-step 1 - :downsample-size 10 - :tournament-size 5 - :umad-rate 0.1 - :variation {:umad 0.5 :crossover 0.5} - :elitism false - :simplification? true - :simplification-steps 100 - :simplification-k 5 - :simplification-verbose? true} - (apply hash-map (map #(if (string? %) (read-string %) %) args)))) + {:instructions instructions + :error-function error-function + :training-data (:train train-and-test-data) + :testing-data (:test train-and-test-data) + :max-generations 500 + :population-size 500 + :max-initial-plushy-size 100 + :step-limit 200 + :parent-selection :lexicase + :tournament-size 5 + :umad-rate 0.1 + :variation {:umad 0.5 :crossover 0.5} + :elitism false} + (apply hash-map (map #(if (string? %) (read-string %) %) args)))) (#?(:clj shutdown-agents))) From be3402dae2e9ecf0f86aa1cea6da8ad99f758619 Mon Sep 17 00:00:00 2001 From: Ryan Boldi Date: Sat, 26 Feb 2022 21:14:16 -0500 Subject: [PATCH 6/6] Revert "autosimplification implementation complete" This reverts commit af22f9793b70339b9c90cfa2b99c327426840471. --- .gitignore | 1 - src/propeller/problems/PSB2/fuel_cost.cljc | 14 -------------- src/propeller/problems/simple_regression.cljc | 0 3 files changed, 15 deletions(-) mode change 100644 => 100755 src/propeller/problems/simple_regression.cljc diff --git a/.gitignore b/.gitignore index 0fc1327..df3fb06 100644 --- a/.gitignore +++ b/.gitignore @@ -33,4 +33,3 @@ node_modules/ # https://github.com/thelmuth/program-synthesis-benchmark-datasets /data **/.DS_Store -.shadow-cljs/classpath.edn diff --git a/src/propeller/problems/PSB2/fuel_cost.cljc b/src/propeller/problems/PSB2/fuel_cost.cljc index 9462ebe..9d45455 100644 --- a/src/propeller/problems/PSB2/fuel_cost.cljc +++ b/src/propeller/problems/PSB2/fuel_cost.cljc @@ -1,7 +1,6 @@ (ns propeller.problems.PSB2.fuel-cost (:require [psb2.core :as psb2] [propeller.genome :as genome] - [propeller.simplification :as simplification] [propeller.push.interpreter :as interpreter] [propeller.utils :as utils] [propeller.push.instructions :refer [get-stack-instructions]] @@ -82,16 +81,3 @@ :elitism false} (apply hash-map (map #(if (string? %) (read-string %) %) args)))) (#?(:clj shutdown-agents))) - - -(defn fuel-cost-autosimplify - [plushy] - (simplification/auto-simplify-plushy {:instructions instructions - :error-function error-function - :training-data (:train train-and-test-data) - :testing-data (:test train-and-test-data) - :max-generations 300 - :population-size 1000 - :max-initial-plushy-size 250 - :step-limit 2000} plushy 5000 error-function (:train train-and-test-data) 5 true)) - diff --git a/src/propeller/problems/simple_regression.cljc b/src/propeller/problems/simple_regression.cljc old mode 100644 new mode 100755