start of fitness function, coming back once I know more

Fix path for regression problem

.gitignore

@@ -23,6 +23,7 @@ notes
 /temp
 *~
 q
+scratch.clj
 
 # Clojure Script
 .shadow-cljs/

README.md

@@ -2,7 +2,25 @@
 
 Yet another Push-based genetic programming system in Clojure.
 
-Full documentation is at [https://lspector.github.io/propeller/](https://lspector.github.io/propeller/).
+Propeller is a component of several active research projects and it is subject to change as a part of these activities.
+
+See the git commit comments for guidance with respect to recent changes.
+
+Some documentation is available at [https://lspector.github.io/propeller/](https://lspector.github.io/propeller/), but this should be evaluated in the context of the commit messages and current source code.
+
+## About this fork
+
+This is mainly for personal use to develop inside of Nix environment. Check out the original on package on [github](https://github.com/lspector/propeller).
+
+### How to jack into the project
+
+Once you've ran `nix develop` and vscodium has appeared, click the REPL
+button in the bottom left of the screen. Click 
+`Start your project with a REPL and connect (a.k.a Jack-in)`, select
+`Leiningen`, and don't select a profile, instead click `OK`. A window should appear to the right with Calva's REPL.
+
+Head over to `src/propeller/session.cljc` as described below and click
+`ctrl + alt + c` then `Enter` to source the session file.
 
 ## Usage
 
@@ -26,15 +44,15 @@ To start a run use `clj -M -m <namespace>` or
 `lein run -m <namespace>`, replacing `<namespace>` 
 with the actual namespace that you will find at the top of the problem file. 
 
-For example, you can run the simple-regression genetic programming problem with:
+For example, you can run the integer-regression genetic programming problem with:
 
 ```
-clj -M -m propeller.problems.simple-regression
+clj -M -m propeller.problems.regression.integer-regression
 ```
 or 
 
 ```
-lein run -m propeller.problems.simple-regression
+lein run -m propeller.problems.regression.integer-regression
 ```
 
 Additional command-line arguments may
@@ -42,13 +60,13 @@ be provided to override the default key/value pairs specified in the
 problem file, for example:
 
 ```
-clj -M -m propeller.problems.simple-regression :population-size 100
+clj -M -m propeller.problems.regression.integer-regression :population-size 100
 ```
 
 or
 
 ```
-lein run -m propeller.problems.simple-regression :population-size 100
+lein run -m propeller.problems.regression.integer-regression :population-size 100
 ```
 
 On Unix operating systems, including MacOS, you can use something
@@ -57,13 +75,13 @@ and to a text file (called `outfile` in this example):
 
 
 ```
-clj -M -m propeller.problems.simple-regression | tee outfile
+clj -M -m propeller.problems.regression.integer-regression | tee outfile
 ```
 
 or
 
 ```
-lein run -m propeller.problems.simple-regression | tee outfile
+lein run -m propeller.problems.regression.integer-regression | tee outfile
 ```
 
 If you want to provide command line arguments that include
@@ -74,13 +92,13 @@ value for the `:variation` argument, which is a clojure map
 containing curly brackets that may confuse your shell:
 
 ```
-clj -M -m propeller.problems.simple-regression :variation "{:umad 1.0}"
+clj -M -m propeller.problems.regression.integer-regression :variation "{:umad 1.0}"
 ```
 
 or
 
 ```
-lein run -m propeller.problems.simple-regression :variation "{:umad 1.0}"
+lein run -m propeller.problems.regression.integer-regression :variation "{:umad 1.0}"
 ```
 
 For many genetic operator hyperparameters, collections may be provided in place of single values. When this is done, a random element of the collection will be chosen (with each being equally likely) each time the operator is used. When specied at the command line, these collections will also have to be quoted, for example with `:umad-rate "[0.01 0.05 0.1]"` to mean that UMAD rates of 0.01, 0.05, and 0.1 can be used.

flake.lock

@@ -0,0 +1,27 @@
+{
+  "nodes": {
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1741600792,
+        "narHash": "sha256-yfDy6chHcM7pXpMF4wycuuV+ILSTG486Z/vLx/Bdi6Y=",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "ebe2788eafd539477f83775ef93c3c7e244421d3",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nixos",
+        "ref": "nixos-24.11",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "nixpkgs": "nixpkgs"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

@@ -0,0 +1,42 @@
+{
+  description = "A flake for developing in a clojure environment with vscode";
+
+  inputs = {
+    nixpkgs.url = "github:nixos/nixpkgs/nixos-24.11";
+  };
+
+  outputs = { nixpkgs, ... }:
+  let system = "x86_64-linux"; in
+  {
+    devShells."${system}".default =
+    let
+      pkgs = nixpkgs.legacyPackages."${system}";
+    in
+      pkgs.mkShellNoCC {
+        buildInputs = [ pkgs.bashInteractive ];
+        packages = with pkgs; [
+          clojure
+          clojure-lsp
+          leiningen
+          nodejs
+          jdk8
+          (vscode-with-extensions.override {
+            vscode = vscodium;
+            vscodeExtensions = pkgs.vscode-utils.extensionsFromVscodeMarketplace [
+              {
+                name = "calva";
+                publisher = "betterthantomorrow";
+                version = "2.0.490";
+                sha256 = "sha256-PCEzSWahrTHXeGFrFShvbXrnoq3AtuVkoohKLxBGDRA=";
+              }
+            ];
+          })
+        ];
+        shellHook = ''
+          export SHELL=${pkgs.lib.getExe pkgs.bashInteractive}
+          echo "starting codium"
+          codium .
+        '';
+      };
+  };
+}

src/docs_src/Additional_Instructions.md

@@ -10,6 +10,7 @@ Table of contents
 * [.DS_Store](#ds_store)
 * [numeric.cljc](#numericcljc)
 * [string.cljc](#stringcljc)
+* [parentheses.cljc](#parenthesescljc)
 * [character.cljc](#charactercljc)
 * [bool.cljc](#boolcljc)
 * [code.cljc](#codecljc)
@@ -24,6 +25,8 @@ Prints new line
 
 # numeric.cljc
 
+## :float_div
+Divides the top two items on the float stack If denominator is 0, returns 1.0 
 ## :float_cos
 Pushes the cosine of the top FLOAT 
 ## :float_sin
@@ -96,6 +99,8 @@ Splits the top STRING on whitespace, and pushes back the resulting components in
 Pushes the substring of the top STRING, with beginning and end indices determined by the second topmost and topmost INTEGERs respectively. If an index is out of bounds, the beginning/end of the string is used instead 
 ## :string_take
 Pushes the substring of the top STRING consisting of its first n letters, where n is determined by the top INTEGER 
+# parentheses.cljc
+
 # character.cljc
 
 ## :char_is_letter

src/propeller/genome.cljc

@@ -2,21 +2,26 @@
   "The genetic material in Propeller. A `plushy` is a list of Push instructions that represent a Push program.
 They hold the genetic material for an `individual`. In the initial population, we create random plushys."
   {:doc/format :markdown}
-  (:require [propeller.push.instructions :as instructions]
+  (:require [propeller.push.instructions.parentheses :as parentheses]
             [propeller.utils :as utils]))
 
 (defn make-random-plushy
-  "Creates and returns a new plushy made of random instructions and of a maximum size of max-initial-plushy-size."
+  "Creates and returns a new plushy made of random instructions."
-  [instructions max-initial-plushy-size]
+  [{:keys [instructions max-initial-plushy-size bmx? bmx-gene-length-limit]
-  (repeatedly
+    :as argmap}]
-   (rand-int max-initial-plushy-size)
+  (let [plushy (repeatedly (rand-int max-initial-plushy-size)
-   #(utils/random-instruction instructions)))
+                           #(utils/random-instruction instructions argmap))]
+    (if bmx?
+      (-> plushy
+          (utils/remove-empty-genes)
+          (utils/enforce-gene-length-limit bmx-gene-length-limit))
+      plushy)))
 
 (defn plushy->push-internal
   [plushy argmap]
   (let [opener? #(and (vector? %) (= (first %) 'open))]    ;; [open <n>] marks opens
     (loop [push ()                                         ;; iteratively build the Push program from the plushy
-           plushy (mapcat #(let [n (get instructions/opens %)]
+           plushy (mapcat #(let [n (get parentheses/opens %)]
                              (if (and n
                                       (> n 0))
                                [% ['open n]]
@@ -48,8 +53,7 @@ They hold the genetic material for an `individual`. In the initial population, w
    (plushy->push plushy {}))
   ;; call plushy->push-internal with possibly-preprocessed plushy
   ([plushy argmap]
-   (plushy->push-internal (if (or (> (or (:ah-umad (:variation argmap)) 0) 0) ;; must strip :vary and :protect
+   (plushy->push-internal (if (:bmx? argmap)
-                                  (> (or (:autoconstructive-crossover (:variation argmap)) 0) 0)) ;; must strip :gene
+                            (filter (complement #{:gap}) plushy)
-                            (filter (complement #{:vary :protect :gene}) plushy)
                             plushy)
                           argmap)))

src/propeller/gp.cljc

@@ -13,6 +13,7 @@
             [propeller.push.instructions.polymorphic]
             [propeller.push.instructions.string]
             [propeller.push.instructions.vector]
+            [propeller.push.instructions.signal]
             [propeller.selection :as selection]
             [propeller.utils :as utils]))
 
@@ -21,142 +22,201 @@
   [evaluations pop generation argmap training-data]
   (let [best (first pop)]
     (utils/pretty-map-println
-     {:generation            generation
+     (merge
-      :best-plushy           (:plushy best)
+      {:generation            generation
-      :best-program          (genome/plushy->push (:plushy best) argmap)
+       :best-plushy           (:plushy best)
-      :best-total-error      (:total-error best)
+       :best-program          (genome/plushy->push (:plushy best) argmap)
-      :evaluations           evaluations
+       :best-total-error      (:total-error best)
-      :ds-indices            (if (:downsample? argmap)
+       :evaluations           evaluations
-                              (map #(:index %) training-data)
+       :ds-indices            (if (:downsample? argmap)
-                               nil)
+                                (map #(:index %) training-data)
-      :best-errors           (:errors best)
+                                nil)
-      :best-behaviors        (:behaviors best)
+       :best-errors           (:errors best)
-      :genotypic-diversity   (float (/ (count (distinct (map :plushy pop))) (count pop)))
+       :best-behaviors        (:behaviors best)
-      :behavioral-diversity  (float (/ (count (distinct (map :behaviors pop))) (count pop)))
+       :genotypic-diversity   (float (/ (count (distinct (map :plushy pop))) (count pop)))
-      :average-genome-length (float (/ (reduce + (map count (map :plushy pop))) (count pop)))
+       :behavioral-diversity  (float (/ (count (distinct (map :behaviors pop))) (count pop)))
-      :average-total-error   (float (/ (reduce + (map :total-error pop)) (count pop)))})))
+       :average-genome-length (float (/ (reduce + (map count (map :plushy pop))) (count pop)))
+       :average-total-error   (float (/ (reduce + (map :total-error pop)) (count pop)))}
+      (if (:bmx? argmap)
+        {:best-gene-count     (utils/count-genes (:plushy best))
+         :average-gene-count  (float (/ (reduce + (map utils/count-genes (map :plushy pop)))
+                                        (count pop)))}
+        {})))
+    (flush)))
 
 (defn cleanup
   []
   #?(:clj (shutdown-agents))
+  (prn {:run-completed true})
   nil)
 
-(defn gp-loop
+(defn fill-defaults
-  "Main GP loop."
+  "Returns argmap with any unspecified values filled with defaults."
-  [{:keys [population-size max-generations error-function instructions
+  [argmap]
-           max-initial-plushy-size solution-error-threshold ds-parent-rate ds-parent-gens dont-end ids-type downsample?]
+  (let [defaults
-    :or   {solution-error-threshold 0.0
+        {:alignment-deviation         2 ; for alternation, the standard deviation of deviation of index when alternating
-           dont-end false
+         :alternation-rate            0.1 ; for alternation, the probability of switching parents at each location
-           ds-parent-rate 0
+         :bmx-exchange-rate           0.5 ; for bmx, the rate at which genes will be exchanged
-           ds-parent-gens 1
+         :bmx-gene-length-limit       10  ; for bmx, the maximum length of a gene
-           ids-type :solved ; :solved or :elite or :soft
+         :bmx-gap-change-probability  0.001 ; for bmx, the mutation rate for gaps
-           downsample? false}
+         :bmx-complementary?          false ; for bmx, whether mates selected using reverse case sequence of first parent
-    :as   argmap}]
+         :bmx-maximum-distance        1000000 ; for bmx, don't exchange if distance is greater than this
-  ;;
+         :bmx-same-gene-count         false ; for bmx, only allow exchanges between individuals with same number of genes
-  (prn {:starting-args (update (update argmap :error-function str) :instructions str)})
+         :closes                      :specified ; :specified, :balanced, :none
-  (println)
+         :custom-report               false ; if provided, should be a function that takes an argmap
-  ;;
+         :dont-end                    false ; if true, keep running until limit regardless of success
-  (loop [generation 0
+         :downsample?                 true ; wether to use downsampling
-         evaluations 0
+         :ds-function                 :case-maxmin ; :case-rand, case-maxmin, case-maxmin-auto
-         population (utils/pmapallv
+         :downsample-rate             0.05 ; proportion of data used in downsample
-                     (fn [_] {:plushy (genome/make-random-plushy instructions max-initial-plushy-size)}) 
+         :ds-parent-rate              0.01 ; proportion of parents used to evaluate case distances
-                     (range population-size) 
+         :ds-parent-gens              10 ; generations between computation of parent distances
-                     argmap)
+         :elitism                     false ; whether always to add the lowest-error individual to the next generation
-         indexed-training-data (if downsample? (downsample/assign-indices-to-data (downsample/initialize-case-distances argmap) argmap) (:training-data argmap))]
+         :error-function              (fn [& args] (println "ERROR FUNCTION NOT PROVIDED")) ; must provide
-    (let [training-data (if downsample?
+         :ids-type                    :solved ; type of informed downsampling, :solved or :elite or :soft
-                          (case (:ds-function argmap)
+         :instructions                ["INSTRUCTIONS NOT PROVIDED"] ; must be provided
-                            :case-maxmin (downsample/select-downsample-maxmin indexed-training-data argmap)
+         :max-batch-size              10 ; for motley-batch-lexicase-selection, the max size of a batch of cases
-                            :case-maxmin-auto (downsample/select-downsample-maxmin-adaptive indexed-training-data argmap)
+         :max-initial-plushy-size     100 ; the maximum size of genomes in initial population
-                            :case-rand (downsample/select-downsample-random indexed-training-data argmap)
+         :max-generations             1000 ; generation limi when downsampling is not used, adjusted by downsampling
-                            (do (prn {:error "Invalid Downsample Function"}) (downsample/select-downsample-random indexed-training-data argmap)))
+         :parent-selection            :lexicase ; see options in variation.cljc
-                          indexed-training-data) ;defaults to full training set
+         :population-size             1000 ; the size of the GP ppopulation
-          parent-reps (if
+         :replacement-rate            0.1 ; for uniform-replacement, the rate at with items will be replaced
-                       (and downsample? ; if we are down-sampling
+         :simplification?             false ; whether to auto-simplify solutions
-                            (zero? (mod generation ds-parent-gens))) ;every ds-parent-gens generations
+         :simplification-k            4 ; when auto-simplifying, max number of items deleted in single step
-                        (take (* ds-parent-rate (count population)) (shuffle population))
+         :simplification-steps        1000 ; when auto-simplifying, number of simplification steps to perform
-                        '()) ;else just empty list
+         :simplification-verbose?     false ; when auto-simplifying, whether to print a lot of information
-          ; parent representatives for down-sampling
+         :single-thread-mode          false ; if true, don't use multithreading
-          rep-evaluated-pop (if downsample?
+         :solution-error-threshold    0 ; maximum total error for solutions
-                              (sort-by :total-error
+         :ssx-not-bmx                 false ; for bmx, swap with segment with same sequence index, not by best match
-                                       (utils/pmapallv
+         :step-limit                  1000 ; limit of Push interpreter steps in a Push program evaluation 
-                                        (partial error-function argmap indexed-training-data)
+         :testing-data                [] ; must be provided unless there is no testing data
-                                        parent-reps
+         :tournament-size             5 ; for torunament selection, the number of individuals in each tournament
-                                        argmap))
+         :training-data               [] ; must be provided
-                              '())
+         :umad-rate                   0.1 ; addition rate (from which deletion rate will be derived) for UMAD
-          evaluated-pop (sort-by :total-error
+         :variation                   {:umad 1} ; genetic operators and probabilities for their use, which should sum to 1
-                                 (utils/pmapallv
+         :tournament-comp-op          min-key ; individual error comparison operator for tournament selection (switch to max-key for max)
-                                  (partial error-function argmap training-data)
+         :lexicase-comp-op            min ; individual error comparison operator for lexicase selection (switch to max for max)
-                                  population
+         :population-sort-comp        < ; comparision operator to use for sorting the population          
-                                  argmap))
+         }
-          best-individual (first evaluated-pop)
+        defaulted (merge defaults argmap)]
-          best-individual-passes-ds (and downsample? (<= (:total-error best-individual) solution-error-threshold))
+    (merge defaulted ; use the map below to include derived values in argmap
-          argmap (if (= (:parent-selection argmap) :epsilon-lexicase)
+           {:bmx? (some #{:bmx :bmx-umad} (keys (:variation defaulted)))})))
-                   (assoc argmap :epsilons (selection/epsilon-list evaluated-pop))
-                   argmap)]   ; epsilons
-      (if (:custom-report argmap)
-        ((:custom-report argmap) evaluations evaluated-pop generation argmap)
-        (report evaluations evaluated-pop generation argmap training-data))
-      ;;did the indvidual pass all cases in ds?
-      (when best-individual-passes-ds
-        (prn {:semi-success-generation generation}))
-      (cond
-        ;; If either the best individual on the ds passes all training cases, or best individual on full sample passes all training cases
-        ;; We verify success on test cases and end evolution
-        (if (or (and best-individual-passes-ds (<= (:total-error (error-function argmap indexed-training-data best-individual)) solution-error-threshold))
-                (and (not downsample?)
-                     (<= (:total-error best-individual) solution-error-threshold)))
-          (do (prn {:success-generation generation})
-              (prn {:total-test-error
-                    (:total-error (error-function argmap (:testing-data argmap) best-individual))})
-              (when (:simplification? argmap)
-                (let [simplified-plushy (simplification/auto-simplify-plushy (:plushy best-individual) error-function argmap)]
-                  (prn {:total-test-error-simplified (:total-error (error-function argmap (:testing-data argmap) (hash-map :plushy simplified-plushy)))})))
-              (if dont-end false true))
-          false)
-        (cleanup)
-        ;;
-        (and (not downsample?) (>= generation max-generations))
-        (cleanup)
-        ;;
-        (and downsample? (>= evaluations (* max-generations population-size (count indexed-training-data))))
-        (cleanup)
-        ;;
-        :else (recur (inc generation)
-                     (+ evaluations (* population-size (count training-data)) ;every member evaluated on the current sample
-                        (if (zero? (mod generation ds-parent-gens)) (* (count parent-reps) (- (count indexed-training-data) (count training-data))) 0) ; the parent-reps not evaluted already on down-sample
-                        (if best-individual-passes-ds (- (count indexed-training-data) (count training-data)) 0)) ; if we checked for generalization or not  
-                     (if (:elitism argmap)
-                          (conj (utils/pmapallv (fn [_] (variation/new-individual evaluated-pop argmap))
-                                                (range (dec population-size))
-                                                argmap)
-                                (first evaluated-pop))         ;elitism maintains the most-fit individual
-                          (utils/pmapallv (fn [_] (variation/new-individual evaluated-pop argmap))
-                                          (range population-size)
-                                          argmap))
-                     (if downsample?
-                       (if (zero? (mod generation ds-parent-gens))
-                         (downsample/update-case-distances rep-evaluated-pop indexed-training-data indexed-training-data ids-type (/ solution-error-threshold (count indexed-training-data))) ; update distances every ds-parent-gens generations
-                         indexed-training-data)
-                       indexed-training-data))))))
 
 (defn gp
-  "Top-level gp function. Calls gp-loop with possibly-adjusted arguments."
+  "Main GP function"
-  [argmap]
+  [non-default-argmap]
-  (let [adjust-for-autoconstructive-crossover
+  (let [argmap (fill-defaults non-default-argmap)
-        (fn [args]
+        {:keys [population-size max-generations error-function solution-error-threshold dont-end
-          (let [prob (:autoconstructive-crossover (:variation args))
+                downsample? ds-parent-rate ds-parent-gens ids-type population-sort-comp]} argmap]
-                n (:autoconstructive-crossover-enrichment args)]
+  ;; print starting args
-            (if (and prob (> prob 0))
+    (prn {:starting-args (update (update argmap :error-function str)
-              (update args :instructions concat (repeat (or n 1) :gene))
+                                 :instructions
-              args)))
+                                 (fn [instrs]
-        ;
+                                   (utils/not-lazy (map #(if (fn? %) (str %) %) instrs))))})
-        adjust-for-ah-umad
+    (println)
-        (fn [args]
+  ;;
-          (let [prob (:ah-umad (:variation args))
+    (loop [generation 0
-                n (:ah-umad-enrichment args)]
+           evaluations 0
-            (if (and prob (> prob 0))
+           population (utils/pmapallv
-              (update args :instructions concat (flatten (repeat (or n 1) [:vary :protect])))
+                       (fn [_] {:plushy (genome/make-random-plushy argmap)})
-              args)))]
+                       (range population-size)
-    (gp-loop (-> argmap
+                       argmap)
-                 (adjust-for-autoconstructive-crossover)
+           indexed-training-data (if downsample?
-                 (adjust-for-ah-umad)))))
+                                   (downsample/assign-indices-to-data (downsample/initialize-case-distances argmap) argmap)
+                                   (:training-data argmap))]
+      (let [training-data (if downsample?
+                            (case (:ds-function argmap)
+                              :case-maxmin (downsample/select-downsample-maxmin indexed-training-data argmap)
+                              :case-maxmin-auto (downsample/select-downsample-maxmin-adaptive indexed-training-data argmap)
+                              :case-rand (downsample/select-downsample-random indexed-training-data argmap)
+                              (do (prn {:error "Invalid Downsample Function"})
+                                  (downsample/select-downsample-random indexed-training-data argmap)))
+                            indexed-training-data) ;defaults to full training set
+            parent-reps (if
+                         (and downsample? ; if we are down-sampling
+                              (zero? (mod generation ds-parent-gens))) ;every ds-parent-gens generations
+                          (take (* ds-parent-rate (count population)) (shuffle population))
+                          '()) ;else just empty list
+          ; parent representatives for down-sampling
+            rep-evaluated-pop (if downsample?
+                                (sort-by :total-error population-sort-comp
+                                         (utils/pmapallv
+                                          (partial error-function argmap indexed-training-data)
+                                          parent-reps
+                                          argmap))
+                                '())
+            evaluated-pop (sort-by :total-error population-sort-comp
+                                   (utils/pmapallv
+                                    (partial error-function argmap training-data)
+                                    population
+                                    argmap))
+            best-individual (first evaluated-pop)
+            best-individual-passes-ds (and downsample? (<= (:total-error best-individual) solution-error-threshold))
+            argmap (if (= (:parent-selection argmap) :epsilon-lexicase)
+                     (assoc argmap :epsilons (selection/epsilon-list evaluated-pop))
+                     argmap)]   ; epsilons
+        (if (:custom-report argmap)
+          ((:custom-report argmap) evaluations evaluated-pop generation argmap)
+          (report evaluations evaluated-pop generation argmap training-data))
+      ;; Did the indvidual pass all cases in ds?
+        (when best-individual-passes-ds
+          (prn {:semi-success-generation generation}))
+        (cond
+        ;; If either the best individual on the ds passes all training cases, or best individual on full 
+        ;; sample passes all training cases, we verify success on test cases and exit, succeeding
+          (if (or (and best-individual-passes-ds
+                       (<= (:total-error (error-function argmap indexed-training-data best-individual))
+                           solution-error-threshold))
+                  (and (not downsample?)
+                       (<= (:total-error best-individual)
+                           solution-error-threshold)))
+            (do (prn {:success-generation generation})
+                (prn {:successful-plushy (:plushy best-individual)})
+                (prn {:successful-program (genome/plushy->push (:plushy best-individual) argmap)})
+                (prn {:total-test-error
+                      (:total-error (error-function argmap (:testing-data argmap) best-individual))})
+                (when (:simplification? argmap)
+                  (let [simplified-plushy (simplification/auto-simplify-plushy (:plushy best-individual) error-function argmap)]
+                    (prn {:total-test-error-simplified
+                          (:total-error (error-function argmap (:testing-data argmap) {:plushy simplified-plushy}))})
+                    (prn {:simplified-plushy simplified-plushy})
+                    (prn {:simplified-program (genome/plushy->push simplified-plushy argmap)})))
+                (if dont-end false true))
+            false)
+          (cleanup)
+        ;; If we've evolved for as many generations as the parameters allow, exit without succeeding
+          (or (and (not downsample?)
+                   (>= generation max-generations))
+              (and downsample?
+                   (>= evaluations (* max-generations population-size (count indexed-training-data)))))
+          (cleanup)
+        ;; Otherwise, evolve for another generation
+          :else (recur (inc generation)
+                       (+ evaluations
+                          (* population-size (count training-data)) ;every member evaluated on the current sample
+                          (if (zero? (mod generation ds-parent-gens))
+                            (* (count parent-reps)
+                               (- (count indexed-training-data)
+                                  (count training-data)))
+                            0) ; the parent-reps not evaluted already on down-sample
+                          (if best-individual-passes-ds
+                            (- (count indexed-training-data) (count training-data))
+                            0)) ; if we checked for generalization or not  
+                       (if (:elitism argmap) ; elitism maintains the individual with lowest total error
+                         (conj (utils/pmapallv (fn [_] (variation/new-individual evaluated-pop argmap))
+                                               (range (dec population-size))
+                                               argmap)
+                               (first evaluated-pop))
+                         (utils/pmapallv (fn [_] (variation/new-individual evaluated-pop argmap))
+                                         (range population-size)
+                                         argmap))
+                       (if downsample?
+                         (if (zero? (mod generation ds-parent-gens))
+                         ; update distances every ds-parent-gens generations
+                           (downsample/update-case-distances rep-evaluated-pop
+                                                             indexed-training-data
+                                                             indexed-training-data
+                                                             ids-type
+                                                             (/ solution-error-threshold
+                                                                (count indexed-training-data)))
+                           indexed-training-data)
+                         indexed-training-data)))))))

src/propeller/problems/PSB2/find_pair.cljc

@@ -77,7 +77,7 @@
                             (math/abs (- (second correct-output) output-2)))))
                     correct-outputs outputs-1 outputs-2)]
     (assoc individual
-           :behavior outputs
+           :behaviors outputs
            :errors errors
            :total-error #?(:clj  (apply +' errors)
                            :cljs (apply + errors)))))

src/propeller/problems/PSB2/solve_boolean.cljc

@@ -49,11 +49,6 @@ Source: https://arxiv.org/pdf/2106.06086.pdf"
                            (:step-limit argmap))
                          :boolean))
                      inputs)
-        parsed-outputs (map (fn [output]
-                              (try (read-string output)
-                                   #?(:clj  (catch Exception e 1000.0)
-                                      :cljs (catch js/Error. e 1000.0))))
-                            outputs)
         errors (map (fn [correct-output output]
                       (if (= output :no-stack-item)
                         10000
@@ -61,9 +56,9 @@ Source: https://arxiv.org/pdf/2106.06086.pdf"
                            0
                            1)))
                     correct-outputs
-                    parsed-outputs)]
+                    outputs)]
     (assoc individual
-      :behaviors parsed-outputs
+      :behaviors outputs
       :errors errors
       :total-error #?(:clj  (apply +' errors)
                       :cljs (apply + errors)))))

src/propeller/problems/boolean/mul3.cljc

@@ -31,9 +31,6 @@
                  (assoc product-bits (keyword (str "c" bit-index)) this-bit)
                  (- remainder (* (if this-bit  1 0) pow2))))))))
 
-#_(target-function false true false false false true)
-#_(target-function true true true true true)
-
 (def train-and-test-data
   (let [bools [false true]]
     {:train (for [a2 bools
@@ -106,7 +103,6 @@
                 [:output :c0]
                 (state/peek-stack state :boolean)))))
 
-
 (def-instruction
   :c5
   ^{:stacks [:boolean :output]}
@@ -227,8 +223,6 @@
         :c1 ;; defined here
         :c0 ;; defined here
 
-        ;; BOOLEAN TAGGING?
-
         ;; Recommended by Kalkreuth et al: BUFa, NOTa, AND, OR, XOR, NAND, NOR, XNOR
         ;:boolean_bufa ;; defined here
         ;:boolean_nota ;; defined here
@@ -300,44 +294,26 @@
   [& args]
   (gp/gp
    (merge
-    {:instructions             instructions
+    {:instructions               instructions
-     :error-function           error-function
+     :error-function             error-function
-     :training-data            (:train train-and-test-data)
+     :training-data              (:train train-and-test-data)
-     :testing-data             (:test train-and-test-data)
+     :testing-data               (:test train-and-test-data)
-     :max-generations          1000
+     :max-generations            1000
-     :population-size          1000
+     :population-size            1000
-     :max-initial-plushy-size  100
+     :max-initial-plushy-size    100
-     :step-limit               10000
+     :step-limit                 10000
-     :parent-selection         :lexicase
+     :parent-selection           :lexicase
-     :downsample?              true
+     :downsample?                false
-     :ds-function              :case-rand
+     :ds-function                :case-rand
-     :downsample-rate          0.1
+     :downsample-rate            0.1
-     ;:parent-selection         :tournament
+     :umad-rate                  0.01
-     ;:parent-selection         :motley-batch-lexicase
+     :variation                  {:umad 0
-     ;:max-batch-size           [1 2 4 8 16 32 64 128 256]
+                                  :bmx 0
-     ;:tournament-size          5
+                                  :bmx-umad 1}
-     :umad-rate                0.01
+     :single-thread-mode         false
-     ;:ah-umad-min              0.01
+     :bmx?                       true
-     ;:ah-umad-max              0.5
+     :bmx-exchange-rate          0.5
-     ;:ah-umad-mean             0.05
+     :bmx-gene-length-limit      10
-     ;:umad-rate                [1/2
+     :bmx-gap-change-probability 0.01
-     ;                           1/4 1/4 
+     :bmx-complementary?         true}
-     ;                           1/8 1/8 1/8  
-     ;                           1/16 1/16 1/16 1/16 
-     ;                           1/32 1/32 1/32 1/32 1/32 
-     ;                           1/64 1/64 1/64 1/64 1/64 1/64 
-     ;                           1/128 1/128 1/128 1/128 1/128 1/128 1/128 
-     ;                           1/256 1/256 1/256 1/256 1/256 1/256 1/256 1/256]
-     ;:alternation-rate         [1 1/2 1/4 1/8 1/16 1/32 1/64 1/128 1/256]
-     ;:alignment-deviation      [0 1 2 4 8 16 32 64 128]
-     :variation                {:ah-umad 0
-                                :umad 0.5
-                                :autoconstructive-crossover 0.5
-                                :alternation 0
-                                :reproduction 0
-                                :tail-aligned-crossover 0}
-     ;:replacement-rate         0.01
-     :elitism                  false
-     :single-thread-mode       false
-     :autoconstructive-crossover-enrichment 10}
     (apply hash-map (map #(if (string? %) (read-string %) %) args)))))

src/propeller/problems/boolean/mul4.cljc

@@ -33,9 +33,6 @@
                  (assoc product-bits (keyword (str "c" bit-index)) this-bit)
                  (- remainder (* (if this-bit  1 0) pow2))))))))
 
-#_(target-function false true false false false true false false)
-#_(target-function true true true true true true true true)
-
 (def train-and-test-data
   (let [bools [false true]]
     {:train (for [a3 bools
@@ -60,12 +57,6 @@
                 [:output :c7]
                 (state/peek-stack state :boolean)))))
 
-#_(interpreter/interpret-program
-   '(:set-c7) (assoc state/empty-state :output {:c7 :unset}) 1000)
-
-#_(interpreter/interpret-program
-   '(true :set-c7) (assoc state/empty-state :output {:c7 :unset}) 1000)
-
 (def-instruction
   :set-c6
   ^{:stacks [:boolean :output]}

src/propeller/problems/data_creation.cljc

@@ -1,8 +1,10 @@
 (ns propeller.problems.data-creation
-  (:require [psb2.core :as psb2]
+  (:require
-            [clojure.data.csv :as csv]
+   [clojure.core :refer [read-string with-open]]
-            [clojure.java.io :as io]
+   [clojure.data.csv :as csv]
-            [clojure.string :as s]))
+   [clojure.java.io :as io]
+   [clojure.string :as s]
+   [psb2.core :as psb2]))
 
 (defn generate-data [problem train-or-test]
   (let [train-and-test-data (psb2/fetch-examples "data" problem 200 1000)

src/propeller/problems/float_regression.cljc

@@ -1,83 +0,0 @@
-(ns propeller.problems.float-regression
-  (:require [propeller.genome :as genome]
-            [propeller.push.interpreter :as interpreter]
-            [propeller.push.state :as state]
-            [propeller.tools.math :as math]
-            [propeller.gp :as gp]
-            #?(:cljs [cljs.reader :refer [read-string]])))
-
-(defn- target-function
-  "Target function: f(x) = (1+ x^3)^3 + 1"
-  [x]
-  (inc (* (inc (* x x x)) (inc (* x x x)) (inc (* x x x)))))
-
-(def train-and-test-data
-  (let [train-inputs (range -1.5 1.5 0.1)
-        test-inputs (range -1.75 1.75 0.05)]
-    {:train (map (fn [x] {:input1 (vector x) :output1 (vector (target-function x))}) train-inputs)
-     :test (map (fn [x] {:input1 (vector x) :output1 (vector (target-function x))}) test-inputs)}))
-
-(def instructions
-  (list :in1
-        :float_add
-        :float_subtract
-        :float_mult
-        :float_quot
-        :float_eq
-        :exec_dup
-        :exec_if
-        'close
-        0.0
-        1.0))
-
-(defn error-function
-  "Finds the behaviors and errors of an individual. The error is the absolute
-  deviation between the target output value and the program's selected behavior,
-  or 1000000 if no behavior is produced. The behavior is here defined as the
-  final top item on the FLOAT stack."
-  ([argmap data individual]
-   (let [program (genome/plushy->push (:plushy individual) argmap)
-         inputs (map (fn [x] (first (:input1 x))) data)
-         correct-outputs (map (fn [x] (first (:output1 x))) data)
-         outputs (map (fn [input]
-                        (state/peek-stack
-                          (interpreter/interpret-program
-                            program
-                            (assoc state/empty-state :input {:in1 input})
-                            (:step-limit argmap))
-                          :float))
-                      inputs)
-         errors (map (fn [correct-output output]
-                       (if (= output :no-stack-item)
-                         1000000
-                         (math/abs (- correct-output output))))
-                     correct-outputs
-                     outputs)]
-     (assoc individual
-       :behaviors outputs
-       :errors errors
-       :total-error #?(:clj  (apply +' errors)
-                       :cljs (apply + errors))))))
-
-(defn -main
-  "Runs the top-level genetic programming function, giving it a map of 
-  arguments with defaults that can be overridden from the command line
-  or through a passed map."
-  [& args]
-  (gp/gp
-    (merge
-      {:instructions             instructions
-       :error-function           error-function
-       :training-data            (:train train-and-test-data)
-       :testing-data             (:test train-and-test-data)
-       :max-generations          300
-       :population-size          1000
-       :max-initial-plushy-size  100
-       :step-limit               200
-       :parent-selection         :epsilon-lexicase
-       :tournament-size          5
-       :umad-rate                0.1
-       :solution-error-threshold  0.5
-       :variation                {:umad 1.0 :crossover 0.0}
-       :elitism                  false}
-      (apply hash-map (map #(if (string? %) (read-string %) %) args)))))

src/propeller/problems/regression/UBall5D.cljc

@@ -1,4 +1,4 @@
-(ns propeller.problems.UBall5D
+(ns propeller.problems.regression.UBall5D
   (:require [propeller.genome :as genome]
             [propeller.push.interpreter :as interpreter]
             [propeller.push.state :as state]
@@ -115,7 +115,7 @@
        :training-data            (:train train-and-test-data)
        :testing-data             (:test train-and-test-data)
        :max-generations          300
-       :population-size          1000
+       :population-size          20
        :max-initial-plushy-size  100
        :step-limit               200
        :parent-selection         :lexicase

src/propeller/problems/regression/integer_regression.cljc

@@ -1,4 +1,4 @@
-(ns propeller.problems.integer-regression
+(ns propeller.problems.regression.integer-regression
   (:require [propeller.genome :as genome]
             [propeller.push.interpreter :as interpreter]
             [propeller.push.state :as state]
@@ -62,6 +62,21 @@
        :total-error #?(:clj  (apply +' errors)
                        :cljs (apply + errors))))))
 
+(def integer-argmap
+  {:instructions             instructions
+   :error-function           error-function
+   :training-data            (:train train-and-test-data)
+   :testing-data             (:test train-and-test-data)
+   :max-generations          300
+   :population-size          1000
+   :max-initial-plushy-size  5
+   :step-limit               200
+   :parent-selection         :lexicase
+   :tournament-size          5
+   :umad-rate                0.1
+   :variation                {:umad 1.0 :crossover 0.0}
+   :elitism                  false})
+
 (defn -main
   "Runs the top-level genetic programming function, giving it a map of 
   arguments with defaults that can be overridden from the command line
@@ -69,17 +84,5 @@
   [& args]
   (gp/gp
     (merge
-      {:instructions             instructions
+     integer-argmap
-       :error-function           error-function
+     (apply hash-map (map #(if (string? %) (read-string %) %) args)))))
-       :training-data            (:train train-and-test-data)
-       :testing-data             (:test train-and-test-data)
-       :max-generations          300
-       :population-size          1000
-       :max-initial-plushy-size  100
-       :step-limit               200
-       :parent-selection         :lexicase
-       :tournament-size          5
-       :umad-rate                0.1
-       :variation                {:umad 1.0 :crossover 0.0}
-       :elitism                  false}
-      (apply hash-map (map #(if (string? %) (read-string %) %) args)))))

src/propeller/problems/regression/nguyen_f1.cljc

@@ -1,4 +1,16 @@
-(ns propeller.problems.complex-regression
+;; This file implements a version of the Nguyen-F1 symbolic regression problem, as described in various
+;; publications in the genetic programming literature including:
+;;
+;;   Makke, N., Chawla, S. Interpretable scientific discovery with symbolic regression: a review. 
+;;   Artif Intell Rev 57, 2 (2024). https://doi.org/10.1007/s10462-023-10622-0
+;;
+;; Note however that it may differ in some respects from the problem used elsewhere, for example
+;; in the data ranges and gentic programming function sets which are not always fully documented
+;; in the literature. For this reason, while this code can be used as an example and for comparing
+;; different configurations of the present system, results obtained with this code may not be directly
+;; comparable to those published in the literature.
+
+(ns propeller.problems.regression.nguyen-f1
   (:require [propeller.genome :as genome]
             [propeller.push.interpreter :as interpreter]
             [propeller.push.state :as state]
@@ -7,14 +19,13 @@
             #?(:cljs [cljs.reader :refer [read-string]])))
 
 (defn- target-function
-  "Target function: f(x) = (x^3 + 1)^3 + 1"
+  "Nguyen F1 = x^3 + x^2 + x"
   [x]
-  (let [x-new (+ (* x x x) 1)]
+  (+ (* x x x) (* x x) x))
-    (+ (* x-new x-new x-new) 1)))
 
 (def train-and-test-data
-  (let [train-inputs (range -4.0 4.0 0.25)
+  (let [train-inputs (range -4.0 4.0 0.1)
-        test-inputs (range -4.125 4.125 0.25)]
+        test-inputs (range -4.0 4.0 0.05)]
     {:train (map (fn [x] {:input1 (vector x) :output1 (vector (target-function x))}) train-inputs)
      :test (map (fn [x] {:input1 (vector x) :output1 (vector (target-function x))}) test-inputs)}))
 
@@ -23,13 +34,12 @@
         :float_add
         :float_subtract
         :float_mult
-        :float_quot
+        :float_div
-        :float_eq
+        :float_sin
-        :exec_dup
+        :float_cos
-        :exec_if
+        :float_tan
-        'close
+        0.0
-        0
+        1.0))
-        1))
 
 (defn error-function
   "Finds the behaviors and errors of an individual. The error is the absolute
@@ -67,20 +77,18 @@
   [& args]
   (gp/gp
    (merge
-    {:instructions             instructions
+    {:instructions              instructions
-     :error-function           error-function
+     :error-function            error-function
-     :training-data            (:train train-and-test-data)
+     :training-data             (:train train-and-test-data)
-     :testing-data             (:test train-and-test-data)
+     :testing-data              (:test train-and-test-data)
-     :max-generations          5000
+     :downsample?               false
-     :population-size          500
+     :solution-error-threshold  0.1
-     :max-initial-plushy-size  100
+     :max-generations           300
-     :step-limit               200
+     :population-size           1000
-     :parent-selection         :ds-lexicase
+     :max-initial-plushy-size   50
-     :ds-function              :case-maxmin
+     :step-limit                100
-     :downsample-rate          0.1
+     :parent-selection          :epsilon-lexicase
-     :case-t-size              20
+     :umad-rate                 0.05
-     :tournament-size          5
+     :variation                 {:umad 1.0}
-     :umad-rate                0.1
+     :simplification?           true}
-     :variation                {:umad 1.0 :crossover 0.0}
-     :elitism                  false}
     (apply hash-map (map #(if (string? %) (read-string %) %) args)))))

src/propeller/problems/regression/nguyen_f5.cljc

@@ -0,0 +1,97 @@
+;; This file implements a version of the Nguyen-F5 symbolic regression problem, as described in various
+;; publications in the genetic programming literature including:
+;;
+;;   Makke, N., Chawla, S. Interpretable scientific discovery with symbolic regression: a review. 
+;;   Artif Intell Rev 57, 2 (2024). https://doi.org/10.1007/s10462-023-10622-0
+;;
+;; Note however that it may differ in some respects from the problem used elsewhere, for example
+;; in the data ranges and gentic programming function sets which are not always fully documented
+;; in the literature. For this reason, while this code can be used as an example and for comparing
+;; different configurations of the present system, results obtained with this code may not be directly
+;; comparable to those published in the literature.
+
+
+(ns propeller.problems.regression.nguyen-f5
+  (:require [propeller.genome :as genome]
+            [propeller.push.interpreter :as interpreter]
+            [propeller.push.state :as state]
+            [propeller.tools.math :as math]
+            [propeller.gp :as gp]
+            #?(:cljs [cljs.reader :refer [read-string]])))
+
+(defn- target-function
+  "Nguyen F5 = sin(x^2)cos(x) - 1"
+  [x]
+  (- (* (Math/sin (* x x))
+        (Math/cos x))
+     1))
+
+(def train-and-test-data
+  (let [train-inputs (range -4.0 4.0 0.1)
+        test-inputs (range -4.0 4.0 0.05)]
+    {:train (map (fn [x] {:input1 (vector x) :output1 (vector (target-function x))}) train-inputs)
+     :test (map (fn [x] {:input1 (vector x) :output1 (vector (target-function x))}) test-inputs)}))
+
+(def instructions
+  (list :in1
+        :float_add
+        :float_subtract
+        :float_mult
+        :float_div
+        :float_sin
+        :float_cos
+        :float_tan
+        0.0
+        1.0))
+
+(defn error-function
+  "Finds the behaviors and errors of an individual. The error is the absolute
+  deviation between the target output value and the program's selected behavior,
+  or 1000000 if no behavior is produced. The behavior is here defined as the
+  final top item on the FLOAT stack."
+  ([argmap data individual]
+   (let [program (genome/plushy->push (:plushy individual) argmap)
+         inputs (map (fn [x] (first (:input1 x))) data)
+         correct-outputs (map (fn [x] (first (:output1 x))) data)
+         outputs (map (fn [input]
+                        (state/peek-stack
+                         (interpreter/interpret-program
+                          program
+                          (assoc state/empty-state :input {:in1 input})
+                          (:step-limit argmap))
+                         :float))
+                      inputs)
+         errors (map (fn [correct-output output]
+                       (if (= output :no-stack-item)
+                         1000000
+                         (math/abs (- correct-output output))))
+                     correct-outputs
+                     outputs)]
+     (assoc individual
+            :behaviors outputs
+            :errors errors
+            :total-error #?(:clj  (apply +' errors)
+                            :cljs (apply + errors))))))
+
+(defn -main
+  "Runs the top-level genetic programming function, giving it a map of 
+  arguments with defaults that can be overridden from the command line
+  or through a passed map."
+  [& args]
+  (gp/gp
+   (merge
+    {:instructions              instructions
+     :error-function            error-function
+     :training-data             (:train train-and-test-data)
+     :testing-data              (:test train-and-test-data)
+     :downsample?               false
+     :solution-error-threshold  0.1
+     :max-generations           300
+     :population-size           1000
+     :max-initial-plushy-size   50
+     :step-limit                100
+     :parent-selection          :epsilon-lexicase
+     :umad-rate                 0.05
+     :variation                 {:umad 1.0}
+     :simplification?           true}
+    (apply hash-map (map #(if (string? %) (read-string %) %) args)))))

src/propeller/problems/stocks/data/GLD.csv

@@ -0,0 +1,763 @@
+77.32,78.28
+78.28,79.57
+79.57,80.1
+80.1,81.3
+81.3,81.25
+81.25,79.36
+79.36,79.47
+79.47,77.24
+77.24,77.75
+77.75,77.95
+77.95,80.28
+80.28,79.12
+79.12,78.3
+78.3,82.18
+82.18,82.23
+82.23,82.24
+82.24,81.43
+81.43,79.75
+79.75,79.83
+79.83,77.93
+77.93,78.62
+78.62,77.35
+77.35,78.12
+78.12,77.69
+77.69,76.03
+76.03,75.52
+75.52,75.23
+75.23,74.6
+74.6,75.7
+75.7,76.0
+76.0,74.5
+74.5,75.12
+75.12,75.14
+75.14,74.59
+74.59,73.91
+73.91,73.36
+73.36,73.09
+73.09,72.53
+72.53,73.4
+73.4,72.87
+72.87,71.9
+71.9,72.35
+72.35,73.9
+73.9,73.51
+73.51,72.7
+72.7,72.0
+72.0,72.33
+72.33,72.28
+72.28,72.34
+72.34,72.72
+72.72,71.43
+71.43,71.7
+71.7,70.97
+70.97,69.99
+69.99,70.08
+70.08,70.46
+70.46,70.52
+70.52,69.62
+69.62,69.39
+69.39,68.86
+68.86,67.56
+67.56,67.44
+67.44,66.52
+66.52,65.8
+65.8,66.07
+66.07,65.59
+65.59,65.98
+65.98,66.11
+66.11,65.31
+65.31,65.4
+65.4,65.07
+65.07,65.12
+65.12,65.01
+65.01,64.68
+64.68,66.13
+66.13,66.29
+66.29,66.26
+66.26,66.57
+66.57,65.46
+65.46,66.77
+66.77,66.48
+66.48,66.52
+66.52,66.69
+66.69,65.89
+65.89,65.93
+65.93,65.79
+65.79,65.77
+65.77,65.41
+65.41,65.65
+65.65,66.9
+66.9,67.47
+67.47,67.47
+67.47,67.58
+67.58,67.01
+67.01,66.62
+66.62,65.75
+65.75,65.82
+65.82,66.03
+66.03,66.02
+66.02,65.44
+65.44,65.61
+65.61,65.38
+65.38,64.96
+64.96,64.42
+64.42,64.74
+64.74,65.02
+65.02,64.27
+64.27,64.26
+64.26,63.67
+63.67,63.62
+63.62,64.43
+64.43,64.78
+64.78,64.56
+64.56,64.71
+64.71,65.48
+65.48,64.95
+64.95,64.85
+64.85,64.6
+64.6,64.53
+64.53,64.1
+64.1,64.7
+64.7,64.22
+64.22,65.26
+65.26,66.41
+66.41,66.37
+66.37,66.54
+66.54,66.44
+66.44,65.54
+65.54,64.72
+64.72,65.07
+65.07,64.94
+64.94,64.75
+64.75,65.54
+65.54,65.2
+65.2,65.67
+65.67,65.52
+65.52,65.06
+65.06,65.6
+65.6,66.54
+66.54,66.28
+66.28,66.45
+66.45,66.0
+66.0,67.45
+67.45,67.88
+67.88,68.25
+68.25,68.19
+68.19,67.49
+67.49,66.66
+66.66,66.69
+66.69,67.09
+67.09,67.56
+67.56,66.87
+66.87,67.89
+67.89,67.73
+67.73,68.26
+68.26,68.7
+68.7,67.53
+67.53,68.38
+68.38,68.0
+68.0,68.4
+68.4,67.84
+67.84,66.99
+66.99,67.08
+67.08,67.16
+67.16,66.53
+66.53,66.86
+66.86,66.81
+66.81,65.83
+65.83,65.85
+65.85,65.74
+65.74,65.65
+65.65,66.05
+66.05,65.7
+65.7,65.84
+65.84,65.15
+65.15,65.76
+65.76,65.82
+65.82,65.27
+65.27,64.74
+64.74,64.62
+64.62,63.98
+63.98,63.94
+63.94,63.72
+63.72,64.37
+64.37,64.25
+64.25,64.48
+64.48,64.3
+64.3,64.15
+64.15,62.93
+62.93,63.71
+63.71,65.82
+65.82,66.48
+66.48,65.41
+65.41,68.1
+68.1,67.72
+67.72,67.15
+67.15,67.35
+67.35,65.31
+65.31,66.36
+66.36,66.41
+66.41,66.37
+66.37,65.84
+65.84,65.67
+65.67,66.12
+66.12,65.52
+65.52,64.63
+64.63,64.79
+64.79,64.31
+64.31,64.28
+64.28,65.22
+65.22,64.83
+64.83,64.22
+64.22,63.77
+63.77,64.17
+64.17,64.06
+64.06,64.33
+64.33,64.23
+64.23,62.72
+62.72,63.0
+63.0,62.26
+62.26,62.64
+62.64,61.97
+61.97,62.17
+62.17,60.63
+60.63,60.59
+60.59,60.85
+60.85,60.48
+60.48,60.17
+60.17,61.65
+61.65,62.28
+62.28,63.21
+63.21,62.9
+62.9,62.22
+62.22,61.98
+61.98,61.65
+61.65,61.38
+61.38,61.62
+61.62,61.8
+61.8,61.04
+61.04,61.0
+61.0,62.12
+62.12,62.48
+62.48,62.52
+62.52,62.57
+62.57,62.05
+62.05,62.73
+62.73,62.69
+62.69,63.81
+63.81,64.12
+64.12,64.12
+64.12,64.39
+64.39,63.16
+63.16,63.57
+63.57,63.7
+63.7,63.5
+63.5,62.5
+62.5,62.27
+62.27,61.77
+61.77,61.78
+61.78,61.31
+61.31,61.84
+61.84,61.63
+61.63,62.19
+62.19,62.49
+62.49,62.95
+62.95,61.11
+61.11,62.04
+62.04,61.89
+61.89,62.3
+62.3,61.92
+61.92,61.36
+61.36,60.24
+60.24,59.9
+59.9,59.4
+59.4,59.29
+59.29,58.76
+58.76,58.17
+58.17,57.76
+57.76,58.78
+58.78,59.41
+59.41,58.59
+58.59,58.65
+58.65,59.17
+59.17,58.57
+58.57,57.49
+57.49,56.88
+56.88,57.0
+57.0,57.2
+57.2,56.99
+56.99,56.92
+56.92,56.37
+56.37,57.11
+57.11,59.15
+59.15,59.47
+59.47,59.79
+59.79,59.81
+59.81,58.74
+58.74,58.5
+58.5,58.5
+58.5,57.95
+57.95,57.26
+57.26,56.92
+56.92,58.23
+58.23,57.4
+57.4,57.19
+57.19,58.62
+58.62,58.32
+58.32,58.5
+58.5,60.64
+60.64,61.25
+61.25,62.86
+62.86,63.36
+63.36,62.14
+62.14,62.29
+62.29,61.42
+61.42,60.94
+60.94,61.0
+61.0,61.77
+61.77,61.75
+61.75,61.83
+61.83,62.05
+62.05,62.44
+62.44,61.04
+61.04,61.03
+61.03,62.49
+62.49,62.01
+62.01,62.25
+62.25,62.71
+62.71,63.25
+63.25,64.63
+64.63,63.97
+63.97,64.5
+64.5,64.28
+64.28,64.12
+64.12,64.75
+64.75,64.32
+64.32,63.16
+63.16,63.11
+63.11,62.9
+62.9,62.0
+62.0,61.55
+61.55,61.14
+61.14,61.71
+61.71,62.54
+62.54,64.03
+64.03,62.9
+62.9,63.95
+63.95,65.85
+65.85,65.55
+65.55,64.92
+64.92,63.81
+63.81,62.02
+62.02,62.63
+62.63,63.02
+63.02,62.5
+62.5,62.18
+62.18,61.23
+61.23,59.52
+59.52,57.54
+57.54,57.68
+57.68,58.28
+58.28,57.99
+57.99,57.73
+57.73,58.33
+58.33,57.3
+57.3,56.36
+56.36,57.68
+57.68,57.32
+57.32,55.62
+55.62,55.92
+55.92,60.03
+60.03,60.45
+60.45,60.91
+60.91,62.28
+62.28,62.55
+62.55,63.29
+63.29,63.5
+63.5,62.56
+62.56,64.23
+64.23,65.11
+65.11,65.1
+65.1,64.7
+64.7,64.06
+64.06,66.38
+66.38,65.3
+65.3,65.58
+65.58,67.46
+67.46,68.15
+68.15,68.61
+68.61,67.41
+67.41,71.12
+71.12,71.03
+71.03,70.38
+70.38,69.68
+69.68,67.56
+67.56,67.99
+67.99,67.48
+67.48,66.46
+66.46,66.55
+66.55,65.16
+65.16,65.09
+65.09,62.96
+62.96,63.65
+63.65,62.78
+62.78,61.65
+61.65,63.2
+63.2,60.96
+60.96,63.75
+63.75,61.85
+61.85,61.09
+61.09,59.5
+59.5,59.43
+59.43,59.01
+59.01,59.6
+59.6,58.59
+58.59,59.28
+59.28,58.66
+58.66,58.33
+58.33,58.47
+58.47,58.1
+58.1,58.6
+58.6,57.07
+57.07,56.06
+56.06,56.39
+56.39,55.71
+55.71,54.7
+54.7,54.75
+54.75,54.83
+54.83,55.17
+55.17,55.12
+55.12,55.34
+55.34,55.12
+55.12,54.87
+54.87,54.31
+54.31,53.83
+53.83,54.24
+54.24,53.97
+53.97,54.95
+54.95,55.23
+55.23,56.28
+56.28,56.74
+56.74,56.1
+56.1,56.07
+56.07,55.22
+55.22,55.64
+55.64,54.64
+54.64,55.34
+55.34,55.12
+55.12,54.96
+54.96,54.52
+54.52,53.76
+53.76,54.49
+54.49,53.72
+53.72,54.79
+54.79,56.18
+56.18,54.83
+54.83,54.6
+54.6,56.72
+56.72,56.5
+56.5,56.98
+56.98,56.7
+56.7,56.7
+56.7,56.68
+56.68,55.63
+55.63,55.88
+55.88,56.14
+56.14,55.73
+55.73,55.77
+55.77,55.2
+55.2,55.59
+55.59,54.07
+54.07,55.26
+55.26,55.44
+55.44,54.43
+54.43,54.53
+54.53,54.06
+54.06,54.6
+54.6,53.72
+53.72,52.34
+52.34,53.3
+53.3,53.12
+53.12,51.58
+51.58,51.47
+51.47,51.47
+51.47,50.64
+50.64,50.12
+50.12,50.14
+50.14,49.37
+49.37,49.04
+49.04,50.22
+50.22,50.09
+50.09,50.24
+50.24,50.35
+50.35,51.65
+51.65,52.56
+52.56,52.4
+52.4,51.9
+51.9,51.32
+51.32,50.89
+50.89,50.78
+50.78,50.32
+50.32,50.17
+50.17,49.1
+49.1,49.8
+49.8,49.74
+49.74,49.42
+49.42,49.15
+49.15,49.32
+49.32,48.99
+48.99,48.46
+48.46,48.48
+48.48,47.78
+47.78,46.66
+46.66,46.62
+46.62,46.8
+46.8,46.48
+46.48,46.56
+46.56,45.91
+45.91,45.77
+45.77,45.51
+45.51,45.96
+45.96,46.13
+46.13,45.73
+45.73,46.4
+46.4,47.22
+47.22,47.18
+47.18,46.93
+46.93,47.06
+47.06,46.39
+46.39,46.52
+46.52,45.98
+45.98,46.34
+46.34,47.0
+47.0,47.26
+47.26,46.8
+46.8,47.02
+47.02,46.85
+46.85,47.39
+47.39,47.37
+47.37,47.36
+47.36,47.21
+47.21,46.36
+46.36,46.45
+46.45,46.42
+46.42,46.7
+46.7,47.1
+47.1,46.8
+46.8,46.25
+46.25,46.57
+46.57,46.28
+46.28,46.36
+46.36,47.06
+47.06,46.23
+46.23,46.25
+46.25,45.82
+45.82,45.42
+45.42,44.96
+44.96,44.56
+44.56,44.89
+44.89,44.84
+44.84,44.6
+44.6,44.33
+44.33,44.27
+44.27,44.25
+44.25,44.22
+44.22,43.4
+43.4,43.01
+43.01,43.57
+43.57,43.61
+43.61,43.73
+43.73,43.59
+43.59,43.76
+43.76,43.7
+43.7,43.6
+43.6,43.83
+43.83,43.91
+43.91,44.55
+44.55,44.13
+44.13,44.49
+44.49,44.45
+44.45,43.65
+43.65,43.34
+43.34,43.35
+43.35,43.6
+43.6,43.68
+43.68,43.53
+43.53,43.14
+43.14,43.13
+43.13,42.82
+42.82,42.75
+42.75,42.44
+42.44,42.25
+42.25,42.51
+42.51,42.45
+42.45,42.46
+42.46,42.2
+42.2,41.89
+41.89,42.01
+42.01,42.05
+42.05,41.9
+41.9,42.35
+42.35,42.64
+42.64,42.54
+42.54,42.26
+42.26,42.36
+42.36,42.25
+42.25,42.28
+42.28,42.7
+42.7,43.44
+43.44,43.63
+43.63,43.45
+43.45,43.91
+43.91,43.9
+43.9,44.02
+44.02,43.72
+43.72,43.81
+43.81,43.66
+43.66,43.63
+43.63,43.46
+43.46,42.74
+42.74,42.6
+42.6,42.78
+42.78,42.62
+42.62,42.27
+42.27,42.34
+42.34,42.38
+42.38,42.48
+42.48,42.17
+42.17,42.1
+42.1,41.53
+41.53,41.65
+41.65,41.88
+41.88,41.69
+41.69,41.87
+41.87,41.73
+41.73,41.62
+41.62,41.65
+41.65,41.98
+41.98,42.05
+42.05,41.86
+41.86,41.88
+41.88,41.95
+41.95,42.16
+42.16,42.69
+42.69,42.64
+42.64,42.55
+42.55,42.52
+42.52,42.91
+42.91,42.9
+42.9,42.76
+42.76,42.88
+42.88,43.35
+43.35,43.0
+43.0,43.2
+43.2,43.64
+43.64,43.44
+43.44,43.39
+43.39,43.19
+43.19,43.4
+43.4,43.28
+43.28,42.69
+42.69,42.4
+42.4,42.31
+42.31,42.9
+42.9,42.84
+42.84,42.78
+42.78,42.69
+42.69,42.57
+42.57,42.63
+42.63,42.45
+42.45,42.4
+42.4,42.62
+42.62,42.82
+42.82,42.61
+42.61,42.56
+42.56,42.54
+42.54,42.39
+42.39,42.42
+42.42,42.65
+42.65,43.09
+43.09,43.89
+43.89,43.82
+43.82,44.31
+44.31,44.06
+44.06,44.03
+44.03,44.43
+44.43,44.2
+44.2,44.02
+44.02,44.03
+44.03,43.47
+43.47,43.38
+43.38,42.97
+42.97,43.25
+43.25,43.22
+43.22,43.52
+43.52,43.5
+43.5,43.33
+43.33,43.42
+43.42,43.57
+43.57,42.75
+42.75,42.73
+42.73,42.52
+42.52,42.58
+42.58,42.55
+42.55,42.08
+42.08,41.75
+41.75,41.31
+41.31,41.26
+41.26,41.27
+41.27,41.47
+41.47,41.68
+41.68,42.16
+42.16,42.1
+42.1,42.22
+42.22,42.69
+42.69,42.62
+42.62,42.69
+42.69,42.24
+42.24,42.77
+42.77,42.74
+42.74,42.25
+42.25,42.26
+42.26,42.32
+42.32,42.32
+42.32,42.6
+42.6,42.6
+42.6,42.21
+42.21,41.95
+41.95,41.84
+41.84,42.15
+42.15,42.67
+42.67,42.74
+42.74,43.02
+43.02,43.8
+43.8,43.83
+43.83,43.66
+43.66,44.37
+44.37,44.48
+44.48,44.27
+44.27,44.02
+44.02,44.22
+44.22,44.34
+44.34,44.19
+44.19,43.68
+43.68,43.99
+43.99,43.55
+43.55,43.91
+43.91,43.44
+43.44,43.78
+43.78,44.01
+44.01,45.11
+45.11,45.17
+45.17,45.6
+45.6,44.95
+44.95,45.38
+45.38,45.12
+45.12,45.4
+45.4,45.29
+45.29,45.05
+45.05,44.75
+44.75,44.95
+44.95,44.78
+44.78,44.38

src/propeller/problems/stocks/data/dailyret_GLD.csv

@@ -0,0 +1,763 @@
+44.38,0.009013068949977434
+44.78,0.003796337650736974
+44.95,-0.004449388209121309
+44.75,0.006703910614525076
+45.05,0.005327413984461754
+45.29,0.0024287922278648584
+45.4,-0.006167400881057294
+45.12,0.005762411347517845
+45.38,-0.009475539885412068
+44.95,0.014460511679644017
+45.6,-0.009429824561403502
+45.17,-0.0013283152534868778
+45.11,-0.02438483706495237
+44.01,-0.005226084980686137
+43.78,-0.007766103243490256
+43.44,0.010819521178637175
+43.91,-0.008198588020951934
+43.55,0.010103329506314693
+43.99,-0.0070470561491248526
+43.68,0.011675824175824131
+44.19,0.0033944331296674743
+44.34,-0.0027063599458729036
+44.22,-0.00452284034373577
+44.02,0.005679236710586097
+44.27,0.004743618703410746
+44.48,-0.0024730215827338
+44.37,-0.01600180302005862
+43.66,0.003893724232707323
+43.83,-0.0006844626967830513
+43.8,-0.017808219178082056
+43.02,-0.006508600650860091
+42.74,-0.001637810014038378
+42.67,-0.01218654792594336
+42.15,-0.007354685646500479
+41.84,0.0026290630975143265
+41.95,0.006197854588796138
+42.21,0.009239516702203283
+42.6,0.0
+42.6,-0.00657276995305167
+42.32,0.0
+42.32,-0.0014177693761815281
+42.26,-0.00023663038334117395
+42.25,0.011597633136094722
+42.74,0.0007019185774450429
+42.77,-0.012391863455693269
+42.24,0.01065340909090899
+42.69,-0.0016397282736003815
+42.62,0.0016424213984045117
+42.69,-0.011009604122745348
+42.22,-0.002842254855518651
+42.1,0.0014251781472682937
+42.16,-0.011385199240986644
+41.68,-0.005038387715930922
+41.47,-0.0048227634434529956
+41.27,-0.00024230676035873796
+41.26,0.0012118274357732493
+41.31,0.010651174049866804
+41.75,0.007904191616766426
+42.08,0.011169201520912522
+42.55,0.0007050528789659492
+42.58,-0.0014091122592765422
+42.52,0.004938852304797595
+42.73,0.0004680552305172742
+42.75,0.019181286549707608
+43.57,-0.0034427358274041445
+43.42,-0.002072777521879397
+43.33,0.003923378721440151
+43.5,0.0004597701149426006
+43.52,-0.006893382352941274
+43.22,0.0006941230911615256
+43.25,-0.006473988439306385
+42.97,0.009541540609727804
+43.38,0.0020746887966804125
+43.47,0.012882447665056413
+44.03,-0.00022711787417665252
+44.02,0.004089050431621983
+44.2,0.005203619909502191
+44.43,-0.009002925950934021
+44.03,0.0006813536225301189
+44.06,0.005674080798910576
+44.31,-0.0110584518167457
+43.82,0.0015974440894568754
+43.89,-0.018227386648439214
+43.09,-0.01021118588999779
+42.65,-0.005392731535756081
+42.42,-0.000707213578500734
+42.39,0.0035385704175512757
+42.54,0.0004701457451810796
+42.56,0.0011748120300751211
+42.61,0.00492842055855435
+42.82,-0.004670714619336825
+42.62,-0.005161895823556989
+42.4,0.0011792452830189685
+42.45,0.00424028268551236
+42.63,-0.0014074595355384066
+42.57,0.002818886539816712
+42.69,0.0021082220660577048
+42.78,0.0014025245441795762
+42.84,0.001400560224089523
+42.9,-0.013752913752913667
+42.31,0.0021271567005435193
+42.4,0.006839622641509414
+42.69,0.013820566877488954
+43.28,0.0027726432532346914
+43.4,-0.004838709677419375
+43.19,0.004630701551285085
+43.39,0.0011523392486747443
+43.44,0.004604051565377598
+43.64,-0.010082493125572817
+43.2,-0.004629629629629695
+43.0,0.008139534883720963
+43.35,-0.01084198385236445
+42.88,-0.0027985074626866733
+42.76,0.0032740879326473475
+42.9,0.00023310023310018672
+42.91,-0.009088790491726718
+42.52,0.0007055503292566799
+42.55,0.0021151586368978476
+42.64,0.0011726078799248864
+42.69,-0.01241508550011715
+42.16,-0.0049810246679315405
+41.95,-0.0016686531585220568
+41.88,-0.00047755491881573843
+41.86,0.004538939321547963
+42.05,-0.0016646848989298523
+41.98,-0.00786088613625532
+41.65,-0.0007202881152461257
+41.62,0.0026429601153291553
+41.73,0.0033549005511622474
+41.87,-0.0042990207786004235
+41.69,0.004557447829215755
+41.88,-0.005491881566380228
+41.65,-0.0028811524609843324
+41.53,0.013725018059234295
+42.1,0.0016627078384798166
+42.17,0.00735119753379168
+42.48,-0.002354048964218322
+42.38,-0.0009438414346389605
+42.34,-0.0016532829475673187
+42.27,0.008280104092737032
+42.62,0.0037541060534960983
+42.78,-0.004207573632538562
+42.6,0.003286384976525835
+42.74,0.016846045858680367
+43.46,0.003911642890013845
+43.63,0.0006876002750399731
+43.66,0.0034356390288594984
+43.81,-0.002054325496462073
+43.72,0.006861848124428277
+44.02,-0.00272603362108143
+43.9,0.00022779043280177701
+43.91,-0.010475973582327346
+43.45,0.00414269275028768
+43.63,-0.004354801741920807
+43.44,-0.017034990791896753
+42.7,-0.00983606557377053
+42.28,-0.0007095553453169616
+42.25,0.002603550295857975
+42.36,-0.002360717658168117
+42.26,0.006625650733554216
+42.54,0.002350728725905064
+42.64,-0.006801125703564708
+42.35,-0.010625737898465238
+41.9,0.003579952267303069
+42.05,-0.0009512485136741772
+42.01,-0.0028564627469649475
+41.89,0.007400334208641735
+42.2,0.006161137440758246
+42.46,-0.00023551577955718348
+42.45,0.001413427561837342
+42.51,-0.00611620795107029
+42.25,0.004497041420118289
+42.44,0.007304429783223428
+42.75,0.001637426900584802
+42.82,0.007239607659972029
+43.13,0.00023185717597955043
+43.14,0.009040333796940208
+43.53,0.0034458993797380788
+43.68,-0.0018315018315017925
+43.6,-0.00573394495412844
+43.35,-0.00023068050749707058
+43.34,0.007152745731425822
+43.65,0.018327605956472034
+44.45,0.0008998875140607232
+44.49,-0.008091706001348604
+44.13,0.009517335146158953
+44.55,-0.014365881032547713
+43.91,-0.001821908449100394
+43.83,-0.005247547342003123
+43.6,0.002293577981651409
+43.7,0.00137299771167037
+43.76,-0.0038848263254112115
+43.59,0.003211745813259772
+43.73,-0.0027441115938714256
+43.61,-0.0009172208209126152
+43.57,-0.01285288042230898
+43.01,0.009067658684026984
+43.4,0.018894009216589867
+44.22,0.0006784260515604057
+44.25,0.00045197740113001413
+44.27,0.0013553196295458586
+44.33,0.00609068351003842
+44.6,0.005381165919282556
+44.84,0.001115075825156047
+44.89,-0.007351303185564676
+44.56,0.00897666068222618
+44.96,0.010231316725978667
+45.42,0.008806693086745896
+45.82,0.009384548232213002
+46.25,-0.00043243243243250003
+46.23,0.017953709712308145
+47.06,-0.014874628134296703
+46.36,-0.0017256255392579442
+46.28,0.006266205704407933
+46.57,-0.006871376422589656
+46.25,0.011891891891891831
+46.8,0.006410256410256502
+47.1,-0.008492569002123113
+46.7,-0.005995717344753771
+46.42,0.0006462731581215238
+46.45,-0.0019375672766416233
+46.36,0.01833477135461608
+47.21,0.003177292946409629
+47.36,0.00021114864864860665
+47.37,0.00042220814861733435
+47.39,-0.011394809031441215
+46.85,0.0036286019210245825
+47.02,-0.004678860059549254
+46.8,0.009829059829059848
+47.26,-0.005501481168006729
+47.0,-0.014042553191489289
+46.34,-0.007768666378938423
+45.98,0.011744236624619537
+46.52,-0.002794496990541757
+46.39,0.014442767837896135
+47.06,-0.0027624309392265734
+46.93,0.005327082889409759
+47.18,0.0008478168715557259
+47.22,-0.01736552308343923
+46.4,-0.01443965517241383
+45.73,0.008746993221080378
+46.13,-0.0036852373726425688
+45.96,-0.00979112271540476
+45.51,0.0057130301032741184
+45.77,0.003058772121476807
+45.91,0.014158135482465818
+46.56,-0.00171821305841936
+46.48,0.0068846815834767705
+46.8,-0.0038461538461538403
+46.62,0.0008580008580008398
+46.66,0.02400342906129457
+47.78,0.014650481372959307
+48.48,-0.00041254125412533047
+48.46,0.010936855138258381
+48.99,0.006736068585425562
+49.32,-0.00344687753446881
+49.15,0.005493387589013288
+49.42,0.006475111290975319
+49.74,0.001206272617611483
+49.8,-0.014056224899598308
+49.1,0.021792260692464364
+50.17,0.002989834562487514
+50.32,0.0091414944356121
+50.78,0.002166207168176436
+50.89,0.008449597170367454
+51.32,0.011301636788776272
+51.9,0.009633911368015415
+52.4,0.0030534351145038875
+52.56,-0.017313546423135533
+51.65,-0.025169409486931214
+50.35,-0.00218470705064547
+50.24,-0.0029856687898088888
+50.09,0.0025953284088639536
+50.22,-0.023496614894464352
+49.04,0.006729200652528513
+49.37,0.01559651610289656
+50.14,-0.00039888312724377996
+50.12,0.010375099760574684
+50.64,0.016390205371247992
+51.47,0.0
+51.47,0.0021371672819117823
+51.58,0.02985653354013182
+53.12,0.0033885542168674647
+53.3,-0.01801125703564716
+52.34,0.026366068016813057
+53.72,0.016381236038719334
+54.6,-0.009890109890109874
+54.06,0.008694043655197907
+54.53,-0.0018338529249954413
+54.43,0.01855594341355866
+55.44,-0.0032467532467532418
+55.26,-0.021534563879840714
+54.07,0.028111707046421363
+55.59,-0.007015650296815984
+55.2,0.010326086956521744
+55.77,-0.0007172314864623678
+55.73,0.007356899336084761
+56.14,-0.004631278945493374
+55.88,-0.004473872584108804
+55.63,0.018874707891425438
+56.68,0.0003528581510233438
+56.7,0.0
+56.7,0.004938271604938166
+56.98,-0.00842400842400837
+56.5,0.003893805309734493
+56.72,-0.03737658674188994
+54.6,0.004212454212454155
+54.83,0.02462155754149191
+56.18,-0.02474190103239588
+54.79,-0.01952911115167002
+53.72,0.014333581533879433
+54.49,-0.013396953569462359
+53.76,0.014136904761904857
+54.52,0.008070432868672005
+54.96,0.0029112081513827616
+55.12,0.003991291727140892
+55.34,-0.01264907842428628
+54.64,0.018301610541727673
+55.64,-0.007548526240115056
+55.22,0.015392973560304264
+56.07,0.0005350454788657238
+56.1,0.011408199643493771
+56.74,-0.008107155445893565
+56.28,-0.018656716417910522
+55.23,-0.005069708491761616
+54.95,-0.01783439490445867
+53.97,0.005002779321845527
+54.24,-0.00755899705014756
+53.83,0.008916960802526547
+54.31,0.010311176578898825
+54.87,0.00455622380171314
+55.12,0.003991291727140892
+55.34,-0.003975424647632923
+55.12,0.0009071117561684374
+55.17,-0.006162769621170988
+54.83,-0.0014590552617180065
+54.75,-0.0009132420091323681
+54.7,0.018464351005484422
+55.71,0.012206067133369228
+56.39,-0.005852101436424868
+56.06,0.018016410988226865
+57.07,0.02680918170667603
+58.6,-0.008532423208191127
+58.1,0.006368330464715963
+58.47,-0.0023943902856165653
+58.33,0.005657466140922309
+58.66,0.010569382884418763
+59.28,-0.011639676113360285
+58.59,0.01723843659327527
+59.6,-0.009899328859060459
+59.01,0.007117437722419958
+59.43,0.0011778563015312179
+59.5,0.02672268907563031
+61.09,0.012440661319364839
+61.85,0.030719482619240072
+63.75,-0.04376470588235293
+60.96,0.036745406824147016
+63.2,-0.024525316455696267
+61.65,0.018329278183292823
+62.78,0.013857916533927962
+63.65,-0.010840534171248983
+62.96,0.03383100381194413
+65.09,0.0010754340144414375
+65.16,0.021332105586249244
+66.55,-0.0013523666416228913
+66.46,0.015347577490219836
+67.48,0.007557794902193107
+67.99,-0.006324459479335088
+67.56,0.0313795145056247
+69.68,0.010045924225028539
+70.38,0.009235578289286811
+71.03,0.0012670702520062426
+71.12,-0.05216535433070877
+67.41,0.017801513128615977
+68.61,-0.006704562017198568
+68.15,-0.010124724871606924
+67.46,-0.027868366439371413
+65.58,-0.0042695943885331065
+65.3,0.016539050535987723
+66.38,-0.034950286230792306
+64.06,0.00999063378083048
+64.7,0.006182380216383176
+65.1,0.00015360983102926447
+65.11,-0.013515589003225241
+64.23,-0.026000311380974646
+62.56,0.015025575447570296
+63.5,-0.0033070866141732417
+63.29,-0.011692210459788308
+62.55,-0.004316546762589865
+62.28,-0.021997430956968603
+60.91,-0.0075521260876702315
+60.45,-0.0069478908188585885
+60.03,-0.06846576711644177
+55.92,-0.005364806866952866
+55.62,0.030564545127651978
+57.32,0.00628053035589671
+57.68,-0.022884882108183083
+56.36,0.016678495386799108
+57.3,0.017975567190226898
+58.33,-0.010286302074404277
+57.73,0.0045037242335008686
+57.99,0.005000862217623713
+58.28,-0.010295126973232694
+57.68,-0.0024271844660194272
+57.54,0.0344108446298228
+59.52,0.028729838709677314
+61.23,0.015515270292340403
+62.18,0.005146349308459316
+62.5,0.00832000000000005
+63.02,-0.006188511583624255
+62.63,-0.009739741338016914
+62.02,0.028861657529829072
+63.81,0.017395392571697216
+64.92,0.009704251386321557
+65.55,0.0045766590389015585
+65.85,-0.028853454821564033
+63.95,-0.016419077404222115
+62.9,0.01796502384737683
+64.03,-0.023270342027174793
+62.54,-0.013271506236008927
+61.71,-0.009236752552260578
+61.14,0.006705920837422254
+61.55,0.007311129163281931
+62.0,0.014516129032258041
+62.9,0.00333863275039747
+63.11,0.0007922674694976574
+63.16,0.01836605446485112
+64.32,0.006685323383084684
+64.75,-0.00972972972972966
+64.12,0.0024953212726137957
+64.28,0.003422526446795253
+64.5,-0.008217054263565908
+63.97,0.010317336251367775
+64.63,-0.021352313167259718
+63.25,-0.008537549407114611
+62.71,-0.007335353213203649
+62.25,-0.00385542168674702
+62.01,0.00774068698597007
+62.49,-0.02336373819811171
+61.03,0.00016385384237257105
+61.04,0.02293577981651374
+62.44,-0.006245996156310067
+62.05,-0.0035455278001611423
+61.83,-0.0012938702895034498
+61.75,0.000323886639676164
+61.77,-0.012465598186822132
+61.0,-0.0009836065573770865
+60.94,0.007876599934361734
+61.42,0.014164767176815328
+62.29,-0.0024080911863862352
+62.14,0.019633086578693254
+63.36,-0.007891414141414142
+62.86,-0.02561247216035634
+61.25,-0.00995918367346938
+60.64,-0.03529023746701848
+58.5,-0.003076923076923072
+58.32,0.005144032921810651
+58.62,-0.024394404640054584
+57.19,0.003671970624235021
+57.4,0.014459930313588821
+58.23,-0.022496994676283622
+56.92,0.005973295853829871
+57.26,0.012050296891372771
+57.95,0.009490940465918847
+58.5,0.0
+58.5,0.004102564102564136
+58.74,0.018215866530473277
+59.81,-0.00033439224210003553
+59.79,-0.005352065562803149
+59.47,-0.005380864301328406
+59.15,-0.034488588334742165
+57.11,-0.012957450534057119
+56.37,0.009756962923540967
+56.92,0.001229796205200286
+56.99,0.0036848569924548317
+57.2,-0.003496503496503546
+57.0,-0.002105263157894692
+56.88,0.010724331926863562
+57.49,0.018785875804487706
+58.57,0.010244152296397498
+59.17,-0.008788237282406677
+58.65,-0.001023017902813217
+58.59,0.01399556238265904
+59.41,-0.010604275374516
+58.78,-0.01735284110241584
+57.76,0.007098337950138568
+58.17,0.010142685232937878
+58.76,0.009019741320626296
+59.29,0.0018552875695732743
+59.4,0.008417508417508417
+59.9,0.005676126878130274
+60.24,0.018592297476759584
+61.36,0.009126466753585435
+61.92,0.006136950904392691
+62.3,-0.0065810593900481
+61.89,0.0024236548715462687
+62.04,-0.01499032882011605
+61.11,0.030109638357061094
+62.95,-0.007307386814932499
+62.49,-0.004800768122899732
+62.19,-0.00900466312912036
+61.63,0.0034074314457244984
+61.84,-0.008570504527813732
+61.31,0.007665959876039779
+61.78,-0.000161864681126546
+61.77,0.008094544277157195
+62.27,0.0036935924201059395
+62.5,0.016
+63.5,0.003149606299212643
+63.7,-0.0020408163265306523
+63.57,-0.006449583136699759
+63.16,0.019474350854971564
+64.39,-0.0041931977015063834
+64.12,0.0
+64.12,-0.0048346849656893676
+63.81,-0.017552107820090965
+62.69,0.0006380602966980244
+62.73,-0.010840108401084007
+62.05,0.008380338436744612
+62.57,-0.0007991050023972696
+62.52,-0.0006397952655151351
+62.48,-0.005761843790012796
+62.12,-0.01802962009014806
+61.0,0.0006557377049180188
+61.04,0.012450851900393153
+61.8,-0.0029126213592232963
+61.62,-0.00389483933787723
+61.38,0.004398826979472076
+61.65,0.005352798053527953
+61.98,0.0038722168441433042
+62.22,0.010928961748633875
+62.9,0.0049284578696343765
+63.21,-0.014712861888941619
+62.28,-0.010115606936416225
+61.65,-0.024006488240064832
+60.17,0.0051520691374438285
+60.48,0.006117724867724943
+60.85,-0.0042728019720624155
+60.59,0.0006601749463607715
+60.63,0.025399967013029837
+62.17,-0.00321698568441375
+61.97,0.01081168307245444
+62.64,-0.006066411238825072
+62.26,0.01188564086090591
+63.0,-0.004444444444444463
+62.72,0.024075255102040897
+64.23,0.0015569048731121642
+64.33,-0.0041971086584796525
+64.06,0.0017171401810802283
+64.17,-0.0062334424185756365
+63.77,0.007056609691077242
+64.22,0.009498598567424469
+64.83,0.006015733456733003
+65.22,-0.014412756823060376
+64.28,0.0004667081518357364
+64.31,0.0074638469911367435
+64.79,-0.002469516900756456
+64.63,0.01377069472381248
+65.52,0.009157509157509288
+66.12,-0.00680580762250458
+65.67,0.0025887010811634185
+65.84,0.008049817739975715
+66.37,0.0006026819346088902
+66.41,-0.0007528986598403427
+66.36,-0.015822784810126538
+65.31,0.03123564538355523
+67.35,-0.0029695619896063646
+67.15,0.008488458674608982
+67.72,0.0056113408151210195
+68.1,-0.03950073421439057
+65.41,0.016358354991591613
+66.48,-0.00992779783393518
+65.82,-0.03205712549377078
+63.71,-0.01224297598493174
+62.93,0.01938662005402838
+64.15,0.002338269680436344
+64.3,0.0027993779160187687
+64.48,-0.0035669975186104833
+64.25,0.0018677042801557128
+64.37,-0.010097871679353825
+63.72,0.003452605147520384
+63.94,0.0006255864873318603
+63.98,0.010003125976867892
+64.62,0.001857010213556025
+64.74,0.00818659252394194
+65.27,0.008426535927684957
+65.82,-0.0009115770282587066
+65.76,-0.009276155717761548
+65.15,0.010590943975441254
+65.84,-0.0021263669501822686
+65.7,0.0053272450532723635
+66.05,-0.006056018168054375
+65.65,0.0013709063214012063
+65.74,0.0016732582902342476
+65.85,-0.00030372057706903606
+65.83,0.014886829712896916
+66.81,0.0007483909594371674
+66.86,-0.004935686509123517
+66.53,0.009469412295205102
+67.16,-0.0011911852293031314
+67.08,-0.0013416815742397646
+66.99,0.012688460964323162
+67.84,0.008254716981132108
+68.4,-0.005847953216374352
+68.0,0.0055882352941175805
+68.38,-0.012430535244223375
+67.53,0.017325633051976923
+68.7,-0.006404657933042179
+68.26,-0.007764430120128935
+67.73,0.0023623209803631563
+67.89,-0.015024304021210723
+66.87,0.010318528488111225
+67.56,-0.006956779159265821
+67.09,-0.005962140408406702
+66.69,-0.00044984255510573004
+66.66,0.012451245124512426
+67.49,0.010371906949177698
+68.19,0.0008798944126705129
+68.25,-0.005421245421245488
+67.88,-0.006334708308780092
+67.45,-0.02149740548554489
+66.0,0.006818181818181861
+66.45,-0.002558314522197166
+66.28,0.003922751961376058
+66.54,-0.01412684099789618
+65.6,-0.00823170731707305
+65.06,0.0070703965570241886
+65.52,0.0022893772893773762
+65.67,-0.007156997106745833
+65.2,0.00521472392638042
+65.54,-0.012053707659444707
+64.75,0.0029343629343628994
+64.94,0.002001847859562603
+65.07,-0.005378822806208611
+64.72,0.01266996291718182
+65.54,0.013732072017088669
+66.44,0.0015051173991572626
+66.54,-0.0025548542230237705
+66.37,0.0006026819346088902
+66.41,-0.01731666917632874
+65.26,-0.015936254980079775
+64.22,0.007474307069448832
+64.7,-0.009273570324575093
+64.1,0.006708268330733336
+64.53,0.0010847667751432385
+64.6,0.0038699690402476785
+64.85,0.001542020046260733
+64.95,0.008160123171670534
+65.48,-0.01175931582162508
+64.71,-0.0023180343069076105
+64.56,0.0034076827757124977
+64.78,-0.0054029021302870375
+64.43,-0.01257178333074669
+63.62,0.0007859163784973949
+63.67,0.009266530548138894
+64.26,0.00015561780267648467
+64.27,0.01166951921580831
+65.02,-0.004306367271608754
+64.74,-0.004942848316342187
+64.42,0.008382489909965726
+64.96,0.006465517241379337
+65.38,0.003517895380850474
+65.61,-0.00259106843468986
+65.44,0.008863080684596552
+66.02,0.0001514692517419739
+66.03,-0.003180372557928335
+65.82,-0.001063506532968599
+65.75,0.013231939163498168
+66.62,0.005854097868507963
+67.01,0.008506193105506538
+67.58,-0.0016277005031074198
+67.47,0.0
+67.47,-0.008448199199644185
+66.9,-0.018684603886397606
+65.65,-0.003655750190403794
+65.41,0.005503745604647599
+65.77,0.000304090010643306
+65.79,0.0021279829761361993
+65.93,-0.0006067040800850333
+65.89,0.012141447867658176
+66.69,-0.0025491078122657326
+66.52,-0.0006013229104027668
+66.48,0.004362214199759206
+66.77,-0.019619589636064135
+65.46,0.01695692025664527
+66.57,-0.004656752290821513
+66.26,0.000452761847268354
+66.29,-0.0024136370493288697
+66.13,-0.021926508392559937
+64.68,0.005102040816326503
+65.01,0.0016920473773265562
+65.12,-0.0007678132678134423
+65.07,0.0050714615029969655
+65.4,-0.0013761467889908778
+65.31,0.012249272699433426
+66.11,-0.0019664196036907494
+65.98,-0.005910882085480457
+65.59,0.007318188748284643
+66.07,-0.004086574844861451
+65.8,0.010942249240121564
+66.52,0.013830426939266413
+67.44,0.0017793594306050498
+67.56,0.01924215512137355
+68.86,0.007696776067383113
+69.39,0.0033145986453380024
+69.62,0.012927319735708006
+70.52,-0.0008508224617130215
+70.46,-0.005393130854385403
+70.08,-0.0012842465753425145
+69.99,0.014002000285755167
+70.97,0.010286036353388811
+71.7,-0.003765690376568982
+71.43,0.018059638807223742
+72.72,-0.005225522552255163
+72.34,-0.0008294166436273469
+72.28,0.0006917542888765517
+72.33,-0.004562422231439213
+72.0,0.009722222222222262
+72.7,0.011141678129298518
+73.51,0.00530540062576521
+73.9,-0.02097428958051436
+72.35,-0.006219765031098668
+71.9,0.013490959666203043
+72.87,0.007273226293399219
+73.4,-0.011852861035422404
+72.53,0.007720943058044978
+73.09,0.0036940757969625942
+73.36,0.007497273718647726
+73.91,0.009200378839128763
+74.59,0.007373642579434202
+75.14,-0.0002661698163427738
+75.12,-0.008253461128860549
+74.5,0.020134228187919462
+76.0,-0.003947368421052594
+75.7,-0.014531043593130892
+74.6,0.008445040214477341
+75.23,0.0038548451415657587
+75.52,0.006753177966101763
+76.03,0.02183348678153356
+77.69,0.005534817865877293
+78.12,-0.009856630824372891
+77.35,0.016418875242404787
+78.62,-0.008776392775375194
+77.93,0.024380854613114223
+79.83,-0.001002129525241116
+79.75,0.02106583072100322
+81.43,0.009947193908878644
+82.24,-0.00012159533073918902
+82.23,-0.0006080505898090375
+82.18,-0.04721343392552944
+78.3,0.01047254150702436
+79.12,0.014661274014155668
+80.28,-0.02902341803687093
+77.95,-0.0025657472738935577
+77.75,-0.006559485530546689
+77.24,0.02887105126877271
+79.47,-0.0013841701270919772
+79.36,0.023815524193548394
+81.25,0.0006153846153845804
+81.3,-0.01476014760147605
+80.1,-0.006616729088639216
+79.57,-0.016212140253864422
+78.28,-0.012263668880940316

src/propeller/problems/stocks/stock_regression.cljc

@@ -0,0 +1,103 @@
+(ns propeller.problems.stocks.stock-regression
+  (:require
+   [clojure.core :refer [read-string]]
+   [propeller.tools.math :as math]
+   [propeller.genome :as genome]
+   [propeller.gp :as gp]
+   [propeller.push.interpreter :as interpreter]
+   [propeller.push.state :as state]
+   [propeller.tools.loading :as loading]
+   [propeller.push.instructions :as instructions]
+   [propeller.utils :as utils]))
+
+(def train-and-test-data
+  "Train and test data split for GLD. Might change this in a bit."
+  (let [data (loading/basic-load-data-csv "src/propeller/problems/stocks/data/dailyret_GLD.csv")
+        train-len (math/round (* (count data) 0.8))
+        test-len (- (count data) train-len)
+        ]
+    {:train (map (fn [x] {:input1 (vector (first (map read-string x))) :output1 (vector (fnext (map read-string x)))}) (doall (take train-len data)))
+     :test (map (fn [x] {:input1 (vector (first (map read-string x))) :output1 (vector (fnext (map read-string x)))}) (doall (take-last test-len data)))}))
+
+;; Random float between -100.0 and 100.0
+(defn random-float "Random float between -100.0 and 100.0" [] (- (* (rand) 200) 100.0))
+
+; Random integer between -100 and 100
+(defn random-int "Random integer between -100 and 100" [] (- (rand-int 201) 100))
+
+(def instructions
+  "stack-specific instructions, input instructions, close, and constants"
+  (utils/not-lazy
+   (concat
+    (instructions/get-stack-instructions #{:float :integer :boolean :signal :exec :code})
+    (list :in1)
+    (list random-float random-int))))
+ 
+(defn error-function
+  "Finds the sharpe ratio for a given individual. Heavy penalty if the stack is empty."
+  ([argmap data individual]
+   (let [program (genome/plushy->push (:plushy individual) argmap)
+         all-inputs (map (fn [x] (first (:input1 x))) data)
+         daily-returns (map (fn [x] (first (:output1 x))) data)
+         ;outputs (map (fn [input]
+         ;               (state/peek-stack
+         ;                 (interpreter/interpret-program
+         ;                   program
+         ;                   (assoc state/empty-state :input {:in1 input})
+         ;                   (:step-limit argmap))
+         ;                 :integer))
+         ;             inputs)
+         ;errors (map (fn [correct-output output]
+         ;              (if (= output :no-stack-item)
+         ;                1000000
+         ;                (math/abs (- correct-output output))))
+         ;            correct-outputs
+         ;            outputs)]
+         ]
+     (loop [sharpes []
+            returns []
+            inputs all-inputs
+            holding-position? false ; true when :buy passes, false when :sell passes
+            trade-signal (if (empty? inputs)
+                           :end-loop
+                           (state/peek-stack
+                            (interpreter/interpret-program
+                             program
+                             (assoc state/empty-state :input {:in1 (first inputs)})
+                             (:step-limit argmap))
+                            :signal))]
+       ; TODO: I'm not confident on calculating the sharpe ratio yet.
+       ; I need time to learn. I'll come back to this when I'm ready.
+       ;(assoc individual
+       ;       :behaviors outputs
+       ;       :errors errors
+       ;       :total-error #?(:clj  (apply +' errors)
+       ;                       :cljs (apply + errors)))
+       ))))
+
+(def stock-argmap
+  {:instructions             instructions
+   :error-function           error-function
+   :training-data            (:train train-and-test-data)
+   :testing-data             (:test train-and-test-data)
+   :max-generations          300
+   :population-size          1000
+   :max-initial-plushy-size  5
+   :step-limit               1000
+   :parent-selection         :lexicase
+   :tournament-size          5
+   :umad-rate                0.1
+   :variation                {:umad 1.0 :crossover 0.0}
+   :elitism                  false
+   :closes                   :balanced
+   :transaction-cost         0.0005})
+
+(defn -main
+  "Runs the top-level genetic programming function, giving it a map of 
+  arguments with defaults that can be overridden from the command line
+  or through a passed map."
+  [& args]
+  (gp/gp
+    (merge
+     stock-argmap
+     (apply hash-map (map #(if (string? %) (read-string %) %) args)))))

src/propeller/push/instructions.cljc

@@ -12,39 +12,6 @@
 can be either constant literals or functions that take and return a Push state"
   (atom (hash-map)))
 
-;; Number of blocks opened by instructions (default = 0)
-(def opens
-  "Number of blocks opened by instructions. The default is 0."
-  {:exec_dup 1
-            :exec_dup_times 1
-            :exec_dup_items 0 ; explicitly set to 0 to make it clear that this is intended
-            :exec_eq 0 ; explicitly set to 0 to make it clear that this is intended
-            :exec_pop 1
-            :exec_rot 3
-            :exec_shove 1
-            :exec_swap 2
-            :exec_yank 0 ; explicitly set to 0 to make it clear that this is intended
-            :exec_yank_dup 0 ; explicitly set to 0 to make it clear that this is intended
-            :exec_deep_dup 0 ; explicitly set to 0 to make it clear that this is intended
-            :exec_print 1
-            :exec_if  2
-            :exec_when 1
-            :exec_while 1
-            :exec_do_while 1
-            :exec_do_range 1
-            :exec_do_count 1
-            :exec_do_times 1
-            :exec_k 2
-            :exec_s 3
-            :exec_y 1
-            :string_iterate 1
-            :vector_boolean_iterate 1
-            :vector_string_iterate 1
-            :vector_integer_iterate 1
-            :vector_float_iterate 1
-            })
-
-
 #?(:clj
    (def ^{:no-doc true} cls->type
      {Boolean    :boolean

src/propeller/push/instructions/numeric.cljc

@@ -20,6 +20,96 @@
   (fn [stack state]
     (make-instruction state > [stack stack] :boolean)))
 
+;; Pushes :buy onto the SIGNAL stack if the first item is greater than the second
+;; item, and acts as a no-op otherwise.
+(def _gt_buy
+  "Pushes :buy onto the SIGNAL stack if the first item is greater
+  than the second item, and acts as a no-op otherwise"
+  ^{:stacks #{}
+    :name "_gt_buy"}
+  (fn [stack state]
+    (make-instruction state #(if (> %1 %2) :buy nil) [stack stack] :signal)))
+
+;; Pushes :sell onto the SIGNAL stack if the first item is greater than the second
+;; item, and acts as a no-op otherwise.
+(def _gt_sell
+  "Pushes :sell onto the SIGNAL stack if the first item is greater
+  than the second item, and acts as a no-op otherwise"
+  ^{:stacks #{}
+    :name "_gt_sell"}
+  (fn [stack state]
+    (make-instruction state #(if (> %1 %2) :sell nil) [stack stack] :signal)))
+
+;; Pushes :hold onto the SIGNAL stack if the first item is greater than the second
+;; item, and acts as a no-op otherwise.
+(def _gt_hold
+  "Pushes :hold onto the SIGNAL stack if the first item is greater
+  than the second item, and acts as a no-op otherwise"
+  ^{:stacks #{}
+    :name "_gt_hold"}
+  (fn [stack state]
+    (make-instruction state #(if (> %1 %2) :hold nil) [stack stack] :signal)))
+
+;; Pushes :buy onto the SIGNAL stack if the first item is greater than the second
+;; item, pushes :sell onto the SIGNAL stack otherwise.
+(def _gt_buy_sell
+  "Pushes :buy onto the SIGNAL stack if the first item is greater
+  than the second item, pushes :sell onto the SIGNAL stack otherwise"
+  ^{:stacks #{}
+    :name "_gt_buy_sell"}
+  (fn [stack state]
+    (make-instruction state #(if (> %1 %2) :buy :sell) [stack stack] :signal)))
+
+;; Pushes :buy onto the SIGNAL stack if the first item is greater than the second
+;; item, pushes :hold onto the SIGNAL stack otherwise.
+(def _gt_buy_hold
+  "Pushes :buy onto the SIGNAL stack if the first item is greater
+  than the second item, pushes :hold onto the SIGNAL stack otherwise"
+  ^{:stacks #{}
+    :name "_gt_buy_hold"}
+  (fn [stack state]
+    (make-instruction state #(if (> %1 %2) :buy :hold) [stack stack] :signal)))
+
+;; Pushes :sell onto the SIGNAL stack if the first item is greater than the second
+;; item, pushes :buy onto the SIGNAL stack otherwise.
+(def _gt_sell_buy
+  "Pushes :sell onto the SIGNAL stack if the first item is greater
+  than the second item, pushes :buy onto the SIGNAL stack otherwise"
+  ^{:stacks #{}
+    :name "_gt_sell_buy"}
+  (fn [stack state]
+    (make-instruction state #(if (> %1 %2) :sell :buy) [stack stack] :signal)))
+
+;; Pushes :sell onto the SIGNAL stack if the first item is greater than the second
+;; item, pushes :hold onto the SIGNAL stack otherwise.
+(def _gt_sell_hold
+  "Pushes :sell onto the SIGNAL stack if the first item is greater
+  than the second item, pushes :hold onto the SIGNAL stack otherwise"
+  ^{:stacks #{}
+    :name "_gt_sell_buy"}
+  (fn [stack state]
+    (make-instruction state #(if (> %1 %2) :sell :hold) [stack stack] :signal)))
+
+;; Pushes :hold onto the SIGNAL stack if the first item is greater than the second
+;; item, pushes :buy onto the SIGNAL stack otherwise.
+(def _gt_hold_buy
+  "Pushes :hold onto the SIGNAL stack if the first item is greater
+  than the second item, pushes :buy onto the SIGNAL stack otherwise"
+  ^{:stacks #{}
+    :name "_gt_hold_buy"}
+  (fn [stack state]
+    (make-instruction state #(if (> %1 %2) :hold :buy) [stack stack] :signal)))
+
+;; Pushes :hold onto the SIGNAL stack if the first item is greater than the second
+;; item, pushes :sell onto the SIGNAL stack otherwise.
+(def _gt_hold_sell
+  "Pushes :hold onto the SIGNAL stack if the first item is greater
+  than the second item, pushes :sell onto the SIGNAL stack otherwise"
+  ^{:stacks #{}
+    :name "_gt_hold_buy"}
+  (fn [stack state]
+    (make-instruction state #(if (> %1 %2) :hold :sell) [stack stack] :signal)))
+
 ;; Pushes TRUE onto the BOOLEAN stack if the second item is greater than or
 ;; equal to the top item, and FALSE otherwise
 (def _gte
@@ -30,6 +120,96 @@
   (fn [stack state]
     (make-instruction state >= [stack stack] :boolean)))
 
+;; Pushes :buy onto the SIGNAL stack if the first item is greater than or equal to the second
+;; item, and acts as a no-op otherwise.
+(def _gte_buy
+  "Pushes :buy onto the SIGNAL stack if the first item is greater
+  than or equal to the second item, and acts as a no-op otherwise"
+  ^{:stacks #{}
+    :name "_gte_buy"}
+  (fn [stack state]
+    (make-instruction state #(if (>= %1 %2) :buy nil) [stack stack] :signal)))
+
+;; Pushes :sell onto the SIGNAL stack if the first item is greater than or equal to the second
+;; item, and acts as a no-op otherwise.
+(def _gte_sell
+  "Pushes :sell onto the SIGNAL stack if the first item is greater
+  than or equal to the second item, and acts as a no-op otherwise"
+  ^{:stacks #{}
+    :name "_gte_sell"}
+  (fn [stack state]
+    (make-instruction state #(if (>= %1 %2) :sell nil) [stack stack] :signal)))
+
+;; Pushes :hold onto the SIGNAL stack if the first item is greater than or equal to the second
+;; item, and acts as a no-op otherwise.
+(def _gte_hold
+  "Pushes :hold onto the SIGNAL stack if the first item is greater
+  than or equal to the second item, and acts as a no-op otherwise"
+  ^{:stacks #{}
+    :name "_gte_hold"}
+  (fn [stack state]
+    (make-instruction state #(if (>= %1 %2) :hold nil) [stack stack] :signal)))
+
+;; Pushes :buy onto the SIGNAL stack if the first item is greater than or equal to the second
+;; item, pushes :sell onto the SIGNAL stack otherwise.
+(def _gte_buy_sell
+  "Pushes :buy onto the SIGNAL stack if the first item is greater
+  than or equal to the second item, pushes :sell onto the SIGNAL stack otherwise"
+  ^{:stacks #{}
+    :name "_gte_buy_sell"}
+  (fn [stack state]
+    (make-instruction state #(if (>= %1 %2) :buy :sell) [stack stack] :signal)))
+
+;; Pushes :buy onto the SIGNAL stack if the first item is greater than or equal to the second
+;; item, pushes :hold onto the SIGNAL stack otherwise.
+(def _gte_buy_hold
+  "Pushes :buy onto the SIGNAL stack if the first item is greater
+  than or equal to the second item, pushes :hold onto the SIGNAL stack otherwise"
+  ^{:stacks #{}
+    :name "_gte_buy_hold"}
+  (fn [stack state]
+    (make-instruction state #(if (>= %1 %2) :buy :hold) [stack stack] :signal)))
+
+;; Pushes :sell onto the SIGNAL stack if the first item is greater than or equal to the second
+;; item, pushes :buy onto the SIGNAL stack otherwise.
+(def _gte_sell_buy
+  "Pushes :sell onto the SIGNAL stack if the first item is greater
+  than or equal to the second item, pushes :buy onto the SIGNAL stack otherwise"
+  ^{:stacks #{}
+    :name "_gte_sell_buy"}
+  (fn [stack state]
+    (make-instruction state #(if (>= %1 %2) :sell :buy) [stack stack] :signal)))
+
+;; Pushes :sell onto the SIGNAL stack if the first item is greater than or equal to the second
+;; item, pushes :hold onto the SIGNAL stack otherwise.
+(def _gte_sell_hold
+  "Pushes :sell onto the SIGNAL stack if the first item is greater
+  than or equal to the second item, pushes :hold onto the SIGNAL stack otherwise"
+  ^{:stacks #{}
+    :name "_gte_sell_buy"}
+  (fn [stack state]
+    (make-instruction state #(if (>= %1 %2) :sell :hold) [stack stack] :signal)))
+
+;; Pushes :hold onto the SIGNAL stack if the first item is greater than or equal to the second
+;; item, pushes :buy onto the SIGNAL stack otherwise.
+(def _gte_hold_buy
+  "Pushes :hold onto the SIGNAL stack if the first item is greater
+  than or equal to the second item, pushes :buy onto the SIGNAL stack otherwise"
+  ^{:stacks #{}
+    :name "_gte_hold_buy"}
+  (fn [stack state]
+    (make-instruction state #(if (>= %1 %2) :hold :buy) [stack stack] :signal)))
+
+;; Pushes :hold onto the SIGNAL stack if the first item is greater than or equal to the second
+;; item, pushes :sell onto the SIGNAL stack otherwise.
+(def _gte_hold_sell
+  "Pushes :hold onto the SIGNAL stack if the first item is greater
+  than or equal to the second item, pushes :sell onto the SIGNAL stack otherwise"
+  ^{:stacks #{}
+    :name "_gte_hold_buy"}
+  (fn [stack state]
+    (make-instruction state #(if (>= %1 %2) :hold :sell) [stack stack] :signal)))
+
 ;; Pushes TRUE onto the BOOLEAN stack if the second item is less than the top
 ;; item, and FALSE otherwise
 (def _lt
@@ -40,6 +220,96 @@
   (fn [stack state]
     (make-instruction state < [stack stack] :boolean)))
 
+;; Pushes :buy onto the SIGNAL stack if the first item is less than the second
+;; item, and acts as a no-op otherwise.
+(def _lt_buy
+  "Pushes :buy onto the SIGNAL stack if the first item is less
+  than the second item, and acts as a no-op otherwise"
+  ^{:stacks #{}
+    :name "_lt_buy"}
+  (fn [stack state]
+    (make-instruction state #(if (< %1 %2) :buy nil) [stack stack] :signal)))
+
+;; Pushes :sell onto the SIGNAL stack if the first item is less than the second
+;; item, and acts as a no-op otherwise.
+(def _lt_sell
+  "Pushes :sell onto the SIGNAL stack if the first item is less
+  than the second item, and acts as a no-op otherwise"
+  ^{:stacks #{}
+    :name "_lt_sell"}
+  (fn [stack state]
+    (make-instruction state #(if (< %1 %2) :sell nil) [stack stack] :signal)))
+
+;; Pushes :hold onto the SIGNAL stack if the first item is less than the second
+;; item, and acts as a no-op otherwise.
+(def _lt_hold
+  "Pushes :hold onto the SIGNAL stack if the first item is less
+  than the second item, and acts as a no-op otherwise"
+  ^{:stacks #{}
+    :name "_lt_hold"}
+  (fn [stack state]
+    (make-instruction state #(if (< %1 %2) :hold nil) [stack stack] :signal)))
+
+;; Pushes :buy onto the SIGNAL stack if the first item is less than the second
+;; item, pushes :sell onto the SIGNAL stack otherwise.
+(def _lt_buy_sell
+  "Pushes :buy onto the SIGNAL stack if the first item is less
+  than the second item, pushes :sell onto the SIGNAL stack otherwise"
+  ^{:stacks #{}
+    :name "_lt_buy_sell"}
+  (fn [stack state]
+    (make-instruction state #(if (< %1 %2) :buy :sell) [stack stack] :signal)))
+
+;; Pushes :buy onto the SIGNAL stack if the first item is less than the second
+;; item, pushes :hold onto the SIGNAL stack otherwise.
+(def _lt_buy_hold
+  "Pushes :buy onto the SIGNAL stack if the first item is less
+  than the second item, pushes :hold onto the SIGNAL stack otherwise"
+  ^{:stacks #{}
+    :name "_lt_buy_hold"}
+  (fn [stack state]
+    (make-instruction state #(if (< %1 %2) :buy :hold) [stack stack] :signal)))
+
+;; Pushes :sell onto the SIGNAL stack if the first item is less than the second
+;; item, pushes :buy onto the SIGNAL stack otherwise.
+(def _lt_sell_buy
+  "Pushes :sell onto the SIGNAL stack if the first item is less
+  than the second item, pushes :buy onto the SIGNAL stack otherwise"
+  ^{:stacks #{}
+    :name "_lt_sell_buy"}
+  (fn [stack state]
+    (make-instruction state #(if (< %1 %2) :sell :buy) [stack stack] :signal)))
+
+;; Pushes :sell onto the SIGNAL stack if the first item is less than the second
+;; item, pushes :hold onto the SIGNAL stack otherwise.
+(def _lt_sell_hold
+  "Pushes :sell onto the SIGNAL stack if the first item is less
+  than the second item, pushes :hold onto the SIGNAL stack otherwise"
+  ^{:stacks #{}
+    :name "_lt_sell_buy"}
+  (fn [stack state]
+    (make-instruction state #(if (< %1 %2) :sell :hold) [stack stack] :signal)))
+
+;; Pushes :hold onto the SIGNAL stack if the first item is less than the second
+;; item, pushes :buy onto the SIGNAL stack otherwise.
+(def _lt_hold_buy
+  "Pushes :hold onto the SIGNAL stack if the first item is less
+  than the second item, pushes :buy onto the SIGNAL stack otherwise"
+  ^{:stacks #{}
+    :name "_lt_hold_buy"}
+  (fn [stack state]
+    (make-instruction state #(if (< %1 %2) :hold :buy) [stack stack] :signal)))
+
+;; Pushes :hold onto the SIGNAL stack if the first item is less than the second
+;; item, pushes :sell onto the SIGNAL stack otherwise.
+(def _lt_hold_sell
+  "Pushes :hold onto the SIGNAL stack if the first item is less
+  than the second item, pushes :sell onto the SIGNAL stack otherwise"
+  ^{:stacks #{}
+    :name "_lt_hold_buy"}
+  (fn [stack state]
+    (make-instruction state #(if (< %1 %2) :hold :sell) [stack stack] :signal)))
+
 ;; Pushes TRUE onto the BOOLEAN stack if the second item is less than or equal
 ;; to the top item, and FALSE otherwise
 (def _lte
@@ -50,6 +320,96 @@
   (fn [stack state]
     (make-instruction state <= [stack stack] :boolean)))
 
+;; Pushes :buy onto the SIGNAL stack if the first item is less than or equal to the second
+;; item, and acts as a no-op otherwise.
+(def _lte_buy
+  "Pushes :buy onto the SIGNAL stack if the first item is less
+  than or equal to the second item, and acts as a no-op otherwise"
+  ^{:stacks #{}
+    :name "_lte_buy"}
+  (fn [stack state]
+    (make-instruction state #(if (<= %1 %2) :buy nil) [stack stack] :signal)))
+
+;; Pushes :sell onto the SIGNAL stack if the first item is less than or equal to the second
+;; item, and acts as a no-op otherwise.
+(def _lte_sell
+  "Pushes :sell onto the SIGNAL stack if the first item is less
+  than or equal to the second item, and acts as a no-op otherwise"
+  ^{:stacks #{}
+    :name "_lte_sell"}
+  (fn [stack state]
+    (make-instruction state #(if (<= %1 %2) :sell nil) [stack stack] :signal)))
+
+;; Pushes :hold onto the SIGNAL stack if the first item is less than or equal to the second
+;; item, and acts as a no-op otherwise.
+(def _lte_hold
+  "Pushes :hold onto the SIGNAL stack if the first item is less
+  than or equal to the second item, and acts as a no-op otherwise"
+  ^{:stacks #{}
+    :name "_lte_hold"}
+  (fn [stack state]
+    (make-instruction state #(if (<= %1 %2) :hold nil) [stack stack] :signal)))
+
+;; Pushes :buy onto the SIGNAL stack if the first item is less than or equal to the second
+;; item, pushes :sell onto the SIGNAL stack otherwise.
+(def _lte_buy_sell
+  "Pushes :buy onto the SIGNAL stack if the first item is less
+  than or equal to the second item, pushes :sell onto the SIGNAL stack otherwise"
+  ^{:stacks #{}
+    :name "_lte_buy_sell"}
+  (fn [stack state]
+    (make-instruction state #(if (<= %1 %2) :buy :sell) [stack stack] :signal)))
+
+;; Pushes :buy onto the SIGNAL stack if the first item is less than or equal to the second
+;; item, pushes :hold onto the SIGNAL stack otherwise.
+(def _lte_buy_hold
+  "Pushes :buy onto the SIGNAL stack if the first item is less
+  than or equal to the second item, pushes :hold onto the SIGNAL stack otherwise"
+  ^{:stacks #{}
+    :name "_lte_buy_hold"}
+  (fn [stack state]
+    (make-instruction state #(if (<= %1 %2) :buy :hold) [stack stack] :signal)))
+
+;; Pushes :sell onto the SIGNAL stack if the first item is less than or equal to the second
+;; item, pushes :buy onto the SIGNAL stack otherwise.
+(def _lte_sell_buy
+  "Pushes :sell onto the SIGNAL stack if the first item is less
+  than or equal to the second item, pushes :buy onto the SIGNAL stack otherwise"
+  ^{:stacks #{}
+    :name "_lte_sell_buy"}
+  (fn [stack state]
+    (make-instruction state #(if (<= %1 %2) :sell :buy) [stack stack] :signal)))
+
+;; Pushes :sell onto the SIGNAL stack if the first item is less than or equal to the second
+;; item, pushes :hold onto the SIGNAL stack otherwise.
+(def _lte_sell_hold
+  "Pushes :sell onto the SIGNAL stack if the first item is less
+  than or equal to the second item, pushes :hold onto the SIGNAL stack otherwise"
+  ^{:stacks #{}
+    :name "_lte_sell_buy"}
+  (fn [stack state]
+    (make-instruction state #(if (<= %1 %2) :sell :hold) [stack stack] :signal)))
+
+;; Pushes :hold onto the SIGNAL stack if the first item is less than or equal to the second
+;; item, pushes :buy onto the SIGNAL stack otherwise.
+(def _lte_hold_buy
+  "Pushes :hold onto the SIGNAL stack if the first item is less
+  than or equal to the second item, pushes :buy onto the SIGNAL stack otherwise"
+  ^{:stacks #{}
+    :name "_lte_hold_buy"}
+  (fn [stack state]
+    (make-instruction state #(if (<= %1 %2) :hold :buy) [stack stack] :signal)))
+
+;; Pushes :hold onto the SIGNAL stack if the first item is less than or equal to the second
+;; item, pushes :sell onto the SIGNAL stack otherwise.
+(def _lte_hold_sell
+  "Pushes :hold onto the SIGNAL stack if the first item is less
+  than or equal to the second item, pushes :sell onto the SIGNAL stack otherwise"
+  ^{:stacks #{}
+    :name "_lte_hold_buy"}
+  (fn [stack state]
+    (make-instruction state #(if (<= %1 %2) :hold :sell) [stack stack] :signal)))
+
 ;; Pushes the sum of the top two items onto the same stack
 (def _add
   "Pushes the sum of the top two items onto the same stack"
@@ -173,11 +533,24 @@ Otherwise, acts as a NOOP"
   (fn [stack state]
     (make-instruction state dec [stack] stack)))
 
-;; 2 types x 16 functions = 32 instructions
+;; Pushes the square of the top item of the stack
+(def _square
+  "Pushes the square of the top item of the stack"
+  ^{:stacks #{}
+    :name "_square"}
+  (fn [stack state]
+    (make-instruction state #(* % %) [stack] stack)))
+
 (generate-instructions
   [:float :integer]
   [_gt _gte _lt _lte _add _subtract _mult _quot _mod _max _min _inc _dec
-   _from_boolean _from_char _from_string])
+   _from_boolean _from_char _from_string _gt_buy _gt_sell _gt_hold _gte_buy
+   _gte_sell _gte_hold _lt_buy _lt_sell _lt_hold _lte_buy _lte_sell _lte_hold
+   _gt_buy_sell _gt_buy_hold _gt_sell_buy _gt_sell_hold _gt_hold_buy _gt_hold_sell
+   _gte_buy_sell _gte_buy_hold _gte_sell_buy _gte_sell_hold _gte_hold_buy _gte_hold_sell
+   _lt_buy_sell _lt_buy_hold _lt_sell_buy _lt_sell_hold _lt_hold_buy _lt_hold_sell
+   _lte_buy_sell _lte_buy_hold _lte_sell_buy _lte_sell_hold _lte_hold_buy _lte_hold_sell
+   _square])
 
 ;; =============================================================================
 ;; FLOAT Instructions only
@@ -212,6 +585,21 @@ Otherwise, acts as a NOOP"
   (fn [state]
     (make-instruction state math/tan [:float] :float)))
 
+;; Pushes the square root of the top FLOAT. Makes the value
+;; absolute first.
+(def-instruction
+  :float_sqrt
+  ^{:stacks #{:float}}
+  (fn [state]
+    (make-instruction state #(math/sqrt (math/abs %)) [:float] :float)))
+
+;; Pushes the natural log of the top FLOAT
+(def-instruction
+  :float_log
+  ^{:stacks #{:float}}
+  (fn [state]
+    (make-instruction state #(if (zero? %) 1 (math/log (math/abs %))) [:float] :float)))
+
 ;; Pushes the floating point version of the top INTEGER
 (def-instruction
   :float_from_integer

src/propeller/push/instructions/parentheses.cljc

@@ -0,0 +1,32 @@
+(ns propeller.push.instructions.parentheses)
+
+;; Number of blocks opened by instructions (default = 0)
+(def opens
+  "Number of blocks opened by instructions. The default is 0."
+  {:exec_dup 1
+   :exec_dup_times 1
+   :exec_dup_items 0 ; explicitly set to 0 to make it clear that this is intended
+   :exec_eq 0 ; explicitly set to 0 to make it clear that this is intended
+   :exec_pop 1
+   :exec_rot 3
+   :exec_shove 1
+   :exec_swap 2
+   :exec_yank 0 ; explicitly set to 0 to make it clear that this is intended
+   :exec_yank_dup 0 ; explicitly set to 0 to make it clear that this is intended
+   :exec_deep_dup 0 ; explicitly set to 0 to make it clear that this is intended
+   :exec_print 1
+   :exec_if  2
+   :exec_when 1
+   :exec_while 1
+   :exec_do_while 1
+   :exec_do_range 1
+   :exec_do_count 1
+   :exec_do_times 1
+   :exec_k 2
+   :exec_s 3
+   :exec_y 1
+   :string_iterate 1
+   :vector_boolean_iterate 1
+   :vector_string_iterate 1
+   :vector_integer_iterate 1
+   :vector_float_iterate 1})

src/propeller/push/instructions/polymorphic.cljc

@@ -232,9 +232,9 @@
         (state/push-to-stack popped-state stack indexed-item))
       state)))
 
-;; 11 types x 13 functions = 143 instructions
+;; 12 types x 13 functions = 156 instructions
 (generate-instructions
   [:boolean :char :code :exec :float :integer :string
-   :vector_boolean :vector_float :vector_integer :vector_string]
+   :vector_boolean :vector_float :vector_integer :vector_string :signal]
   [_dup _dup_times _dup_items _empty _eq _flush _pop _rot _shove
    _stack_depth _swap _yank _yank_dup _deep_dup])

src/propeller/push/instructions/signal.cljc

@@ -0,0 +1,49 @@
+(ns propeller.push.instructions.signal
+  "SIGNAL instructions. Instructions pertaining only to trading signals
+   are located here."
+  (:require [propeller.tools.math :as math]
+           [propeller.push.instructions :refer [def-instruction
+                                                generate-instructions
+                                                make-instruction]]))
+
+;; Changes the top signal from :buy to :sell if applicable
+(def-instruction
+  :signal_swap_buy_sell
+  ^{:stacks #{:signal}}
+  (fn [state]
+    (make-instruction state #(if (= :buy %) :sell :buy) [:signal] :signal)))
+
+;; Changes the top signal from :buy to :hold if applicable
+(def-instruction
+  :signal_swap_buy_hold
+  ^{:stacks #{:signal}}
+  (fn [state]
+    (make-instruction state #(if (= :buy %) :hold :buy) [:signal] :signal)))
+
+;; Changes the top signal from :sell to :buy if applicable
+(def-instruction
+  :signal_swap_sell_buy
+  ^{:stacks #{:signal}}
+  (fn [state]
+    (make-instruction state #(if (= :sell %) :buy :sell) [:signal] :signal)))
+
+;; Changes the top signal from :sell to :hold if applicable
+(def-instruction
+  :signal_swap_sell_hold
+  ^{:stacks #{:signal}}
+  (fn [state]
+    (make-instruction state #(if (= :sell %) :hold :sell) [:signal] :signal)))
+
+;; Changes the top signal from :hold to :buy if applicable
+(def-instruction
+  :signal_swap_hold_buy
+  ^{:stacks #{:signal}}
+  (fn [state]
+    (make-instruction state #(if (= :hold %) :buy :hold) [:signal] :signal)))
+
+;; Changes the top signal from :hold to :sell if applicable
+(def-instruction
+  :signal_swap_hold_sell
+  ^{:stacks #{:signal}}
+  (fn [state]
+    (make-instruction state #(if (= :hold %) :sell :hold) [:signal] :signal)))

src/propeller/push/interpreter.cljc

@@ -11,6 +11,11 @@
         instruction (first (:exec state))
         literal-type (instructions/get-literal-type instruction)]     ; nil for non-literals
     (cond
+      ;;
+      ;; Catch the trading signal keywords before the other
+      ;; keyword checks.
+      (some #(= instruction %) '(:buy :sell :hold))
+      (state/push-to-stack popped-state :signal instruction)
       ;;
       ;; Recognize functional instruction or input instruction
       (keyword? instruction)

src/propeller/push/state.cljc

@@ -5,19 +5,21 @@
 
 ;; Empty push state - all available stacks are empty
 (defonce ^:no-doc empty-state {:boolean        '()
-                      :char           '()
+                               :char           '()
-                      :code           '()
+                               :code           '()
-                      :exec           '()
+                               :exec           '()
-                      :float          '()
+                               :float          '()
-                      :input          {}
+                               :input          {}
-                      :output         {}
+                               :output         {}
-                      :integer        '()
+                               :integer        '()
-                      :print          '("")
+                               :print          '("")
-                      :string         '()
+                               :string         '()
-                      :vector_boolean '()
+                               :vector_boolean '()
-                      :vector_float   '()
+                               :vector_float   '()
-                      :vector_integer '()
+                               :vector_integer '()
-                      :vector_string  '()})
+                               :vector_string  '()
+                               :signal         '() ;; stock trading signal (:buy, :sell, or :hold only), long only for now
+                               })
 
 ;; All stack types available in a Push state
 (defonce ^:no-doc stacks (set (keys empty-state)))

src/propeller/selection.cljc

@@ -11,7 +11,7 @@
   [pop argmap]
   (let [tournament-size (:tournament-size argmap)
         tournament-set (take tournament-size (shuffle pop))]
-    (apply min-key :total-error tournament-set)))
+    (apply (:tournament-comp-op argmap) :total-error tournament-set)))
 
 (defn lexicase-selection
   "Selects an individual from the population using lexicase selection.
@@ -19,16 +19,18 @@
   eliminating any individuals with errors for the current case that are worse than the best error in the selection pool,
   until a single individual remains."
   [pop argmap]
-  (loop [survivors (map rand-nth (vals (group-by :errors pop)))
+  (let [initial-cases (or (:initial-cases argmap)
-         cases (shuffle (range (count (:errors (first pop)))))]
+                          (shuffle (range (count (:errors (first pop))))))]
-    (if (or (empty? cases)
+    (loop [survivors (map rand-nth (vals (group-by :errors pop)))
-            (empty? (rest survivors)))
+           cases initial-cases]
-      (rand-nth survivors)
+      (if (or (empty? cases)
-      (let [min-err-for-case (apply min (map #(nth % (first cases))
+              (empty? (rest survivors)))
-                                             (map :errors survivors)))]
+        (assoc (rand-nth survivors) :selection-cases initial-cases)
-        (recur (filter #(= (nth (:errors %) (first cases)) min-err-for-case)
+        (let [err-for-case (apply (:lexicase-comp-op argmap) (map #(nth % (first cases))
-                       survivors)
+                                               (map :errors survivors)))]
-               (rest cases))))))
+          (recur (filter #(= (nth (:errors %) (first cases)) err-for-case)
+                         survivors)
+                 (rest cases)))))))
 
 (defn fitness-proportionate-selection 
   "Selects an individual from the population using a fitness proportionate selection."
@@ -92,12 +94,12 @@
       (if (or (empty? cases)
               (empty? (rest survivors)))
         (rand-nth survivors)
-        (let [min-err-for-case (apply min (map #(nth % (first cases))
+        (let [err-for-case (apply (:lexicase-comp-op argmap) (map #(nth % (first cases))
                                                (map :errors survivors)))
               epsilon (nth epsilons (first cases))]
           (recur (filter #(<= (Math/abs (- (nth (:errors %)
                                                 (first cases))
-                                           min-err-for-case))
+                                           err-for-case))
                               epsilon)
                          survivors)
                  (rest cases)))))))

src/propeller/session.cljc

@@ -16,7 +16,30 @@
             [propeller.variation :as variation]
             [propeller.push.instructions :as instructions]
             [propeller.push.interpreter :as interpreter]
-            [propeller.push.state :as state]))
+            [propeller.push.state :as state]
+            [propeller.problems.regression.integer-regression :as regression]
+            [propeller.problems.string-classification :as sc]
+            [propeller.downsample :as downsample]
+            ))
+
+(def test-argmap-to-fill
+  {:instructions             regression/instructions
+   :error-function           regression/error-function
+   :training-data            (:train regression/train-and-test-data)
+   :testing-data             (:test regression/train-and-test-data)
+   :max-generations          300
+   :population-size          1000
+   :max-initial-plushy-size  5
+   :step-limit               200
+   :parent-selection         :lexicase
+   :tournament-size          5
+   :umad-rate                0.1
+   :variation                {:umad 1.0 :crossover 0.0}
+   :elitism                  false
+   :downsample-rate          0.5})
+
+(def test-argmap (gp/fill-defaults test-argmap-to-fill))
+
 
 ;; Interpreting a simple Push program:
 
@@ -31,9 +54,10 @@
 ;; A program with a conditional:
 
 #_(interpreter/interpret-program
-    '(3 3 :integer_eq :exec_if (1 "yes") (2 "no"))
+   '(3 3 :integer_eq :exec_if (1 "yes") (2 "no"))
-    state/empty-state
+   ;;(assoc state/empty-state :keep-history true)
-    1000)
+   state/empty-state
+   1000)
 
 ;; A program using an input instruction:
 
@@ -55,29 +79,35 @@
 ;; and returning the Push program expressed by the genome:
 
 #_(genome/plushy->push
-    (genome/make-random-plushy (instructions/get-stack-instructions #{:float :integer :exec :boolean}) 20))
+    (genome/make-random-plushy 
+     {:instructions (instructions/get-stack-instructions #{:float :integer :exec :boolean}) 
+      :max-initial-plushy-size 100 
+      :bmx? true
+      :bmx-gene-length-limit 10}))
 
 ;; One way of running a genetic programming problem defined in the project
 ;; is to require the problem's namespace and then call `gp/gp` using the
 ;; items defined for the problem. Depending on your IDE and setup, you may
 ;; also have to open the problem's file and evaluate its contents.
 
-#_(require '[propeller.problems.simple-regression :as regression])
+;;#_(require '[propeller.problems.simple-regression :as regression])
 
 #_(gp/gp {:instructions            regression/instructions
         :error-function          regression/error-function
         :training-data           (:train regression/train-and-test-data)
         :testing-data            (:test regression/train-and-test-data)
-        :max-generations         500
+        :max-generations         20
-        :population-size         500
+        :population-size         50
         :max-initial-plushy-size 100
         :step-limit              200
-        :parent-selection        :tournament
+        :parent-selection        :lexicase
         :tournament-size         5
         :umad-rate               0.01
         :variation               {:umad      1.0
                                   :crossover 0.0}
-        :elitism                 false})
+        :elitism                 false
+        :downsample?             false
+          })
 
 #_(require '[propeller.problems.string-classification :as sc])
 
@@ -86,7 +116,7 @@
           :training-data           (:train sc/train-and-test-data)
           :testing-data            (:test sc/train-and-test-data)
           :max-generations         500
-          :population-size         500
+          :population-size         15
           :max-initial-plushy-size 100
           :step-limit              200
           :parent-selection        :lexicase
@@ -107,3 +137,16 @@
 #_(require '[propeller.problems.simple-regression :as regression])
 #_(regression/-main :population-size 100 :variation {:umad 1.0})
 
+;; Analyze how downsample indicies are assigned
+#_(downsample/assign-indices-to-data (:train regression/train-and-test-data) regression/integer-argmap)
+
+;; How does initializing the case distances work?
+regression/integer-argmap
+#_(downsample/initialize-case-distances regression/integer-argmap)
+
+;; How does select-downsample-maxmin work?
+#_(downsample/select-downsample-maxmin 
+   (downsample/assign-indices-to-data 
+    (downsample/initialize-case-distances test-argmap) 
+    test-argmap)
+   test-argmap)

src/propeller/tools/loading.cljc

@@ -0,0 +1,12 @@
+(ns propeller.tools.loading
+  "Functions to help facilitate loading data"
+  (:require
+   [clojure.core :refer [with-open]]
+   [clojure.java.io :as io]
+   [clojure.data.csv :as csv]))
+
+(defn basic-load-data-csv
+  [path]
+  (with-open [reader (io/reader path)]
+    (->> (csv/read-csv reader)
+         (doall))))

src/propeller/tools/metrics.cljc

@@ -1,6 +1,7 @@
 (ns propeller.tools.metrics
   "Functions to measure things."
-  (:require [propeller.tools.math :as math]))
+  (:require [clojure.set]
+            [propeller.tools.math :as math]))
 
 (defn argmins
   "returns the indice(s) of the minimum value of a list. Could be more efficient, probably"
@@ -124,3 +125,16 @@
     (let [distance (levenshtein-distance seq1 seq2)
           max-distance (max (count seq1) (count seq2))]
       (math/div (- max-distance distance) max-distance))))
+
+(defn multiset-distance
+  "Returns the total of the differences between the counts of all items across 
+   the provided multisets."
+  [ms1 ms2]
+  (loop [total 0
+         remaining (clojure.set/union (set ms1) (set ms2))]
+    (if (empty? remaining)
+      total
+      (recur (+ total
+                (math/abs (- (count (filter (partial = (first remaining)) ms1))
+                             (count (filter (partial = (first remaining)) ms2)))))
+             (rest remaining)))))

src/propeller/utils.cljc

@@ -1,7 +1,10 @@
 (ns propeller.utils
   "Useful functions."
   (:require [clojure.zip :as zip]
-            [clojure.repl :as repl]))
+            [clojure.repl :as repl]
+            [propeller.tools.metrics :as metrics]
+            [propeller.tools.math :as math]
+            [propeller.push.instructions.parentheses :as parentheses]))
 
 (defn filter-by-index
   "filters a collection by a list of indices"
@@ -47,12 +50,31 @@
 (defn random-instruction
   "Returns a random instruction from a supplied pool of instructions, evaluating
   ERC-producing functions to a constant literal."
-  [instructions]
+  [instructions argmap]
-  (let [instruction (rand-nth instructions)]
+  (case (or (:closes argmap) :specified) 
-    (if (fn? instruction)
+    :specified (let [instruction (rand-nth instructions)]
-      (instruction)
+                 (if (fn? instruction)
-      instruction)))
+                   (instruction)
-
+                   instruction))
+    :balanced (let [source (remove #(= % 'close) instructions)
+                    p (/ (apply + (filter identity
+                                          (map #(get parentheses/opens %) source)))
+                         (count source))]
+                (if (< (rand) p)
+                  'close
+                  (let [instruction (rand-nth source)]
+                    (if (fn? instruction)
+                      (instruction)
+                      instruction))))
+    :none (let [multi-block-instructions (set (filter (fn [i]
+                                                        (let [opens (get parentheses/opens i)]
+                                                          (and opens (> opens 1))))
+                                                      instructions))
+                source (remove (set (conj multi-block-instructions 'close)) instructions)
+                instruction (rand-nth source)]
+            (if (fn? instruction)
+              (instruction)
+              instruction))))
 
 (defn count-points
   "Returns the number of points in tree, where each atom and each pair of parentheses
@@ -162,3 +184,84 @@
     (doseq [[k v] (rest mp-seq)]
       (println (str " " (pr-str k v)))))
   (println "}"))
+
+(defn count-genes
+  "A utility for best match crossover (bmx). Returns the number of segments 
+   between (and before and after) instances of :gap."
+  [plushy]
+  (inc (count (filter #(= % :gap) plushy))))
+
+(defn extract-genes
+  "A utility for best match crossover (bmx). Returns the segments of the plushy
+   before/between/after instances of :gap."
+  [plushy]
+  (loop [genes []
+         current-gene []
+         remainder plushy]
+    (cond (empty? remainder)
+          (conj genes current-gene)
+          ;
+          (= (first remainder) :gap)
+          (recur (conj genes current-gene)
+                 []
+                 (rest remainder))
+          ;
+          :else
+          (recur genes
+                 (conj current-gene (first remainder))
+                 (rest remainder)))))
+
+(defn bmx-distance
+  "A utility function for bmx. Returns the distance between two plushies
+   computed as half of their multiset-distance plus their length difference."
+  [p1 p2]
+  (+ (* 0.5 (metrics/multiset-distance p1 p2))
+     (math/abs (- (count p1) (count p2)))))
+
+(defn remove-empty-genes
+  "A utility function for bmx-related genetic operators. Returns the provided
+   plushy with any empty genes (regions before/between/after instances of :gap)
+   removed."
+  [plushy]
+  (vec (flatten (interpose :gap (filter #(not (empty? %))
+                                        (extract-genes plushy))))))
+
+(defn break-up
+  "A utility function for bmx-related genetic operators. Returns the provided
+   :gap-free plushy with gaps randomly inserted to ensure that no gene is longer
+   than the provided limit. Will break just before an instruction that opens code
+   blocks or just after a close, unless there are no opportunities to do so"
+  [gene limit]
+  (if (> (count gene) limit)
+    (let [openers (map first
+                       (filter #(and (second %)
+                                     (not (zero? (second %))))
+                               parentheses/opens))
+          i (if (or (some (set openers) (rest gene))
+                    (some #{'close} (butlast gene)))
+              (rand-nth (filter identity
+                                (concat (map-indexed (fn [ix item]
+                                                       (if (some #{item} openers)
+                                                         (inc ix)
+                                                         nil))
+                                                     (rest gene))
+                                        (map-indexed (fn [ix item]
+                                                       (if (= item 'close)
+                                                         (inc ix)
+                                                         nil))
+                                                     (butlast gene)))))
+              (inc (rand-int (dec (count gene)))))]
+      (concat (break-up (take i gene) limit)
+              [:gap]
+              (break-up (drop i gene) limit)))
+    gene))
+
+(defn enforce-gene-length-limit
+  "A utility function for bmx-related genetic operators. Returns the provided
+   plushy with any over-length genes broken into non-empty pieces, recursively
+   until all genes obey the limit."
+  [plushy limit]
+  (flatten (interpose :gap
+                      (mapv (fn [gene]
+                              (break-up gene limit))
+                            (extract-genes plushy)))))