circleci · tanya-coding · Feb 27, 2019 · Feb 26, 2019
@@ -7,17 +7,20 @@
     [clj-stacktrace.core :refer (parse-trace-elem)]
     [clj-stacktrace.repl :refer (method-str)]
     [circleci.rollcage.json :as json]
-    [circleci.rollcage.throwables :as throwables])
+    [circleci.rollcage.throwables :as throwables]
+    [clojure.walk :as walk])
   (:import
-    [java.net InetAddress UnknownHostException]
-    [java.util UUID]))
+   [java.net InetAddress UnknownHostException]
+   [java.util UUID]
+   (clojure.lang Keyword)))
 
 (def ^:private endpoint "https://api.rollbar.com/api/1/item/")
 
 (def ^:private http-conn-timeout 3000)
 (def ^:private http-socket-timeout 3000)
 
 (def ^:private Client {:access-token (s/maybe String)
+                       :block-fields (s/maybe [s/Keyword])
                        :result-fn clojure.lang.IFn
                        :send-fn clojure.lang.IFn
                        :data {:environment (s/maybe String)
@@ -164,25 +167,26 @@
 
 (s/defn ^:private client* :- Client
   [access-token :- (s/maybe String)
-   {:keys [os hostname environment code-version file-root result-fn]
+   {:keys [os hostname environment code-version file-root result-fn block-fields]
     :or {environment "production"}}]
   (let [os        (or os (guess-os))
         hostname  (or hostname (guess-hostname))
         file-root (or file-root (guess-file-root))
         result-fn (or result-fn (constantly nil))]
     {:access-token access-token
      :result-fn result-fn
+     :block-fields block-fields
      :send-fn (if (string/blank? access-token)
                 send-item-null
                 send-item-http)
      :data {:environment (name environment)
-            :platform    (name os)
-            :language    "Clojure"
-            :framework   "Ring"
-            :notifier    {:name "Rollcage"}
-            :server      {:host hostname
-                          :root file-root
-                          :code_version code-version}}}))
+            :platform (name os)
+            :language "Clojure"
+            :framework "Ring"
+            :notifier {:name "Rollcage"}
+            :server {:host hostname
+                     :root file-root
+                     :code_version code-version}}}))
 
 (defn client
   "Create a client that can can be passed used to send notifications to Rollbar.
@@ -212,7 +216,7 @@
   There is no default value.
 
   :result-fn
-  An function that will be called after each exception is sent to Rollbar.
+  A function that will be called after each exception is sent to Rollbar.
   The function will be passed 2 parameters:
   - The Throwable that was being reported
   - A map with the result of sending the exception to Rollbar. This map will
@@ -221,6 +225,15 @@
                  Rollbar, 0 otherwise.
       :message - A human-readable message describing the error.
 
+  :block-fields
+  A list of fields to remove/scrub from the payload prior to sending to Rollbar
+  using kebab case keywords. Input can contain keys in any variation of kebab or
+  snake cased keywords or strings.
+  For example, given :first-name field the following keys will be automatically
+  removed from input:
+    :first-name :first_name \"first-name\" \"first_name\"
+  Example: [:first-name :last-name :address]
+
   See https://rollbar.com/docs/api/items_post/
 
   More information on System Properties:
@@ -230,13 +243,40 @@
   ([access-token options]
    (client* access-token options)))
 
+(defn- fields-to-scrub
+  [block-fields]
+  (-> block-fields
+      (concat (map json/snake-case block-fields))
+      (concat (map name block-fields))
+      (concat (map (comp keyword json/snake-case) block-fields))))
+
+(defn- scrub-map [a-map block-fields]
+  (reduce
+   (fn remove-field [acc field]
+     (if (get acc field)
+       (assoc acc field "*field removed*")
+       acc))
+   a-map
+   block-fields))
+
+(defn scrub [item block-fields]
+  (let [all-fields (fields-to-scrub block-fields)]
+    (if (seq all-fields)
+      (walk/postwalk
+       (fn [form]
+         (if (map? form)
+           (scrub-map form all-fields)
+           form))
+       item)
+      item)))
+
 (defn notify
   ([^String level client ^Throwable exception]
    (notify level client exception {}))
-  ([^String level {:keys [result-fn  send-fn] :as client} ^Throwable exception {:keys [url params]}]
-   (let [log-level (rollbar-to-logging level)
-         params (merge params (throwables/merged-ex-data exception))
-         item (make-rollbar client level exception url params)
+  ([^String level {:keys [result-fn send-fn block-fields] :as client} ^Throwable exception {:keys [url params]}]
+   (let [params (merge params (throwables/merged-ex-data exception))
+         scrubbed (scrub params block-fields)
+         item (make-rollbar client level exception url scrubbed)
          result (try
                   (send-fn endpoint exception item)
                   (catch Exception e

@@ -1,7 +1,6 @@
 (ns circleci.rollcage.test-core
   (:use clojure.test)
-  (:require [bond.james :as bond]
-            [schema.test :refer (validate-schemas)]
+  (:require [schema.test :refer (validate-schemas)]
             [clojure.test :refer :all]
             [clojure.string :as string]
             [circleci.rollcage.core :as client]
@@ -102,13 +101,16 @@
   (let [c (client/client "access-token")]
     (is (= "access-token" (:access-token c))))
   (let [c (client/client "access-token" {:os "DOS"
-                                         :environment "alpha"})]
+                                         :environment "alpha"
+                                         :block-fields [:foo :bar]})]
     (is (= "access-token" (:access-token c)))
     (is (= "alpha" (-> c :data :environment)))
-    (is (= "DOS" (-> c :data :platform))))
-
-  (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Output of client\* does not match schema"
-                        (client/client "e" {:hostname 1}))))
+    (is (= "DOS" (-> c :data :platform)))
+    (is (= [:foo :bar] (:block-fields c))))
+  (is (thrown-with-msg? ExceptionInfo #"Output of client\* does not match schema"
+                        (client/client "e" {:hostname 1})))
+  (is (thrown-with-msg? ExceptionInfo #"Output of client\* does not match schema"
+                        (client/client "e" {:block-fields ["foo"]}))))
 
 (deftest environments-can-be-kw-or-string
   (letfn [(env [e] (-> (client/client "token" {:environment e}) :data :environment))]
@@ -168,11 +170,11 @@
                  @delivery-exceptions)))))))
 
 (deftest ^:integration it-can-send-ex-data
-  (let [token (System/getenv "ROLLBAR_ACCESS_TOKEN")
-        cause (Exception. "connection error")
-        e (ex-info "system error" {:key1 "one" :key2 "two"} cause)]
+  (let [token (System/getenv "ROLLBAR_ACCESS_TOKEN")]
     (testing "it can send items"
-      (let [r (client/client token {:code-version "9d95d17105b4e752c46ccf656aaefad5ace50699"})
+      (let [cause (Exception. "connection error")
+            e (ex-info "system error" {:key1 "one" :key2 "two"} cause)
+            r (client/client token {:code-version "9d95d17105b4e752c46ccf656aaefad5ace50699"})
             {err :err {uuid :uuid} :result} (client/warning r e)]
         (is (zero? err))
         (is (UUID/fromString uuid))))))
@@ -202,7 +204,7 @@
             {err :err skipped :skipped {uuid :uuid} :result} (client/warning r e)]
         (is (zero? err))
         (is (true? skipped))
-        (is (UUID/fromString uuid))))))
+        (is (and uuid (UUID/fromString uuid)))))))
 
 (deftest it-reports-ex-data
   (let [p (promise)
@@ -215,5 +217,37 @@
     (is (not (= result :failed)))
     (is (= {:foo 1 :bar 2} (get-in result [:data :custom])))))
 
+(deftest it-scrubs-ex-data
+  (let [p (promise)
+        client (assoc (client/client "access-token" {:block-fields [:foo :other-foo]})
+                      :send-fn (fn [_ _ item]
+                                 (deliver p item)
+                                 {:err 0}))
+        _ (client/critical client (ex-info "outer" {:foo 1} (ex-info "inner" {:bar 2})))
+        result (deref p 0 :failed)]
+    (is (not (= result :failed)))
+    (is (= {:foo "*field removed*" :bar 2} (get-in result [:data :custom])))))
+
+(deftest scrub-works
+  (is (= {:first_name "*field removed*"}
+         (client/scrub {:first_name "john"} [:first-name])))
+
+  (is (= {:first_name "*field removed*" "last_name" "*field removed*"}
+         (client/scrub {:first_name "john" "last_name" "bobson"} [:first-name :last-name])))
+
+  (is (= {:first_name "john"}
+         (client/scrub {:first_name "john"} nil)))
+
+  (is (= {:first_name "john"}
+         (client/scrub {:first_name "john"} [])))
+
+  (is (= {:payment-profile "*field removed*"}
+         (client/scrub {:payment-profile {:first_name "john"}} [:payment-profile]))
+      "Will remove entire object")
+
+  (is (= {:first_name "*field removed*" (type "aaa") (type 90)}
+         (client/scrub {:first_name (type 24) (type "aaa") (type 90)} [:first-name]))
+      "Will not fail when keys or values cannot be JSON encoded"))
+
 (comment
   (run-tests))