Add tests for the normalize transform

closes #473

Author: Fil

src/options.js

@@ -307,6 +307,14 @@ export function isTemporal(values) {
   }
 }
 
+export function isWeaklyNumeric(values) {
+  for (const value of values) {
+    if (value == null) continue;
+    if (typeof value === "number") return true; // note: includes NaN!
+    return !isNaN(+value);
+  }
+}
+
 // Are these strings that might represent dates? This is stricter than ISO 8601
 // because we want to ignore false positives on numbers; for example, the string
 // "1192" is more likely to represent a number than a date even though it is

src/transforms/group.js

@@ -29,7 +29,8 @@ import {
   range,
   second,
   percentile,
-  isTemporal
+  isTemporal,
+  isWeaklyNumeric
 } from "../options.js";
 import {basic} from "./basic.js";
 
@@ -253,7 +254,7 @@ export function maybeReduce(reduce, value) {
     case "proportion-facet":
       return reduceProportion(value, "facet");
     case "deviation":
-      return reduceAccessor(deviation);
+      return reduceNumbers(deviation);
     case "min":
       return reduceAccessor(min);
     case "min-index":
@@ -267,7 +268,7 @@ export function maybeReduce(reduce, value) {
     case "median":
       return reduceMaybeTemporalAccessor(median);
     case "variance":
-      return reduceAccessor(variance);
+      return reduceNumbers(variance);
     case "mode":
       return reduceAccessor(mode);
     case "x":
@@ -322,9 +323,19 @@ function reduceAccessor(f) {
   };
 }
 
+function reduceNumbers(f) {
+  return {
+    reduce(I, X) {
+      if (!isWeaklyNumeric(X)) throw new Error("non-numeric data");
+      return f(I, (i) => X[i]);
+    }
+  };
+}
+
 function reduceMaybeTemporalAccessor(f) {
   return {
     reduce(I, X) {
+      if (!isWeaklyNumeric(X)) throw new Error("non-numeric data");
       const x = f(I, (i) => X[i]);
       return isTemporal(X) ? new Date(x) : x;
     }
@@ -385,7 +396,7 @@ const reduceDistinct = {
   }
 };
 
-const reduceSum = reduceAccessor(sum);
+const reduceSum = reduceNumbers(sum);
 
 function reduceProportion(value, scope) {
   return value == null

src/transforms/normalize.js

@@ -1,6 +1,6 @@
 import {extent, deviation, max, mean, median, min, sum} from "d3";
 import {defined} from "../defined.js";
-import {percentile, take} from "../options.js";
+import {isWeaklyNumeric, percentile, take} from "../options.js";
 import {mapX, mapY} from "./map.js";
 
 /** @jsdoc normalizeX */
@@ -46,6 +46,7 @@ export function normalize(basis) {
 function normalizeBasis(basis) {
   return {
     map(I, S, T) {
+      if (!isWeaklyNumeric(S)) throw new Error("non-numeric data");
       const b = +basis(I, S);
       for (const i of I) {
         T[i] = S[i] === null ? NaN : S[i] / b;
@@ -60,6 +61,7 @@ function normalizeAccessor(f) {
 
 const normalizeExtent = {
   map(I, S, T) {
+    if (!isWeaklyNumeric(S)) throw new Error("non-numeric data");
     const [s1, s2] = extent(I, (i) => S[i]),
       d = s2 - s1;
     for (const i of I) {
@@ -69,13 +71,15 @@ const normalizeExtent = {
 };
 
 const normalizeFirst = normalizeBasis((I, S) => {
+  if (!isWeaklyNumeric(S)) throw new Error("non-numeric data");
   for (let i = 0; i < I.length; ++i) {
     const s = S[I[i]];
     if (defined(s)) return s;
   }
 });
 
 const normalizeLast = normalizeBasis((I, S) => {
+  if (!isWeaklyNumeric(S)) throw new Error("non-numeric data");
   for (let i = I.length - 1; i >= 0; --i) {
     const s = S[I[i]];
     if (defined(s)) return s;
@@ -84,6 +88,7 @@ const normalizeLast = normalizeBasis((I, S) => {
 
 const normalizeDeviation = {
   map(I, S, T) {
+    if (!isWeaklyNumeric(S)) throw new Error("non-numeric data");
     const m = mean(I, (i) => S[i]);
     const d = deviation(I, (i) => S[i]);
     for (const i of I) {

src/transforms/window.js

@@ -1,6 +1,6 @@
 import {deviation, max, min, median, mode, variance} from "d3";
 import {defined} from "../defined.js";
-import {percentile, take} from "../options.js";
+import {isWeaklyNumeric, percentile, take} from "../options.js";
 import {warn} from "../warnings.js";
 import {mapX, mapY} from "./map.js";
 
@@ -96,6 +96,7 @@ function reduceNumbers(f) {
     strict
       ? {
           map(I, S, T) {
+            if (!isWeaklyNumeric(S)) throw new Error("non-numeric data");
             const C = Float64Array.from(I, (i) => (S[i] === null ? NaN : S[i]));
             let nans = 0;
             for (let i = 0; i < k - 1; ++i) if (isNaN(C[i])) ++nans;
@@ -108,6 +109,7 @@ function reduceNumbers(f) {
         }
       : {
           map(I, S, T) {
+            if (!isWeaklyNumeric(S)) throw new Error("non-numeric data");
             const C = Float64Array.from(I, (i) => (S[i] === null ? NaN : S[i]));
             for (let i = -s; i < 0; ++i) {
               T[I[i + s]] = f(C.subarray(0, i + k));
@@ -149,6 +151,7 @@ function reduceSum(k, s, strict) {
   return strict
     ? {
         map(I, S, T) {
+          if (!isWeaklyNumeric(S)) throw new Error("non-numeric data");
           let nans = 0;
           let sum = 0;
           for (let i = 0; i < k - 1; ++i) {
@@ -169,6 +172,7 @@ function reduceSum(k, s, strict) {
       }
     : {
         map(I, S, T) {
+          if (!isWeaklyNumeric(S)) throw new Error("non-numeric data");
           let sum = 0;
           const n = I.length;
           for (let i = 0, j = Math.min(n, k - s - 1); i < j; ++i) {
@@ -188,6 +192,7 @@ function reduceMean(k, s, strict) {
     const sum = reduceSum(k, s, strict);
     return {
       map(I, S, T) {
+        if (!isWeaklyNumeric(S)) throw new Error("non-numeric data");
         sum.map(I, S, T);
         for (let i = 0, n = I.length - k + 1; i < n; ++i) {
           T[I[i + s]] /= k;
@@ -197,6 +202,7 @@ function reduceMean(k, s, strict) {
   } else {
     return {
       map(I, S, T) {
+        if (!isWeaklyNumeric(S)) throw new Error("non-numeric data");
         let sum = 0;
         let count = 0;
         const n = I.length;
@@ -248,6 +254,7 @@ function reduceDifference(k, s, strict) {
   return strict
     ? {
         map(I, S, T) {
+          if (!isWeaklyNumeric(S)) throw new Error("non-numeric data");
           for (let i = 0, n = I.length - k; i < n; ++i) {
             const a = S[I[i]];
             const b = S[I[i + k - 1]];
@@ -257,6 +264,7 @@ function reduceDifference(k, s, strict) {
       }
     : {
         map(I, S, T) {
+          if (!isWeaklyNumeric(S)) throw new Error("non-numeric data");
           for (let i = -s, n = I.length - k + s + 1; i < n; ++i) {
             T[I[i + s]] = lastNumber(S, I, i, k) - firstNumber(S, I, i, k);
           }
@@ -268,6 +276,7 @@ function reduceRatio(k, s, strict) {
   return strict
     ? {
         map(I, S, T) {
+          if (!isWeaklyNumeric(S)) throw new Error("non-numeric data");
           for (let i = 0, n = I.length - k; i < n; ++i) {
             const a = S[I[i]];
             const b = S[I[i + k - 1]];
@@ -277,6 +286,7 @@ function reduceRatio(k, s, strict) {
       }
     : {
         map(I, S, T) {
+          if (!isWeaklyNumeric(S)) throw new Error("non-numeric data");
           for (let i = -s, n = I.length - k + s + 1; i < n; ++i) {
             T[I[i + s]] = lastNumber(S, I, i, k) / firstNumber(S, I, i, k);
           }

test/transforms/normalize-test.js

@@ -41,6 +41,16 @@ it("Plot.normalize deviation doesn’t crash on equal values", () => {
   testNormalize([1, 1], "deviation", [0, 0]);
 });
 
+it("normalizeX throws on non-numeric values", () => {
+  const data = [null, "A", 10, 8];
+  testNormalizeThrows(data);
+  testNormalizeThrows(data, "first");
+  testNormalizeThrows(data, "last");
+  testNormalizeThrows(data, "mean");
+  testNormalizeThrows(data, "sum");
+  testNormalizeThrows(data, "deviation");
+});
+
 function testNormalize(data, basis, r) {
   const mark = Plot.dot(data, Plot.normalizeY(basis, {y: data}));
   const {
@@ -50,3 +60,8 @@ function testNormalize(data, basis, r) {
   } = mark.initialize();
   assert.deepStrictEqual(Y, r);
 }
+
+function testNormalizeThrows(data, basis) {
+  const mark = Plot.dot(data, Plot.normalizeX({x: data, basis}));
+  assert.throws(() => mark.initialize());
+}

test/transforms/reduce-test.js

@@ -10,6 +10,14 @@ it("baked-in reducers reduce as expected", () => {
   testReducer(data, "min", 0);
   testReducer(data, "sum", 21);
   testReducer(data, "variance", 10.7);
+  testReducer(data, "mode", 0);
+});
+
+it("baked-in non-numeric reducers throw on non-numeric data", () => {
+  const data = ["A", "B", "C", "B"];
+  testReducer(data, "min", "A");
+  testReducer(data, "max", "C");
+  testReducer(data, "mode", "B");
 });
 
 it("function reducers reduce as expected", () => {
@@ -18,12 +26,39 @@ it("function reducers reduce as expected", () => {
   testReducer(data, (v) => v.join(", "), "0, 1, 2, 4, 5, 9");
 });
 
+it.only("baked-in numeric reducers throw on non-numeric data", () => {
+  const data = [null, "A", 1, 2, 3];
+  testReducerThrows(data, "deviation");
+  testReducerThrows(data, "mean");
+  testReducerThrows(data, "median");
+  testReducerThrows(data, "sum");
+  testReducerThrows(data, "variance");
+});
+
+it("baked-in numeric reducers accept and return temporal data", () => {
+  const data = [null, new Date(2001, 0, 1), new Date(2002, 0, 3), 0];
+  testReducer(data, "mean", new Date("1991-01-01T23:20:00.000Z"));
+  testReducer(data, "median", new Date("2000-12-31T23:00:00.000Z"));
+});
+
+it("baked-in numeric reducers accept temporal data", () => {
+  const data = [null, new Date(2001, 0, 1), new Date(2002, 0, 2)];
+  testReducer(data, "deviation", 22360413477.393482);
+  testReducer(data, "sum", 1988229600000);
+  testReducer(data, "variance", 499988090880000000000);
+});
+
 function testReducer(data, x, r) {
-  const mark = Plot.dot(data, Plot.groupZ({x}, {x: (d) => d}));
+  const mark = Plot.dot(data, Plot.groupZ({x}, {x: data}));
   const {
     channels: {
       x: {value: X}
     }
   } = mark.initialize();
   assert.deepStrictEqual(X, [r]);
 }
+
+function testReducerThrows(data, x) {
+  const mark = Plot.dot(data, Plot.groupZ({x}, {x: data}));
+  assert.throws(() => mark.initialize());
+}

test/transforms/window-test.js

@@ -311,3 +311,54 @@ it(`windowX({reduce: "last", k}) does not coerce to numbers`, () => {
   const m3 = applyTransform(Plot.windowX({reduce: "last", k: 3, x: (d) => d}), data);
   assert.deepStrictEqual(m3.x.transform(), ["B", "A", "A", "C", "C", "A", "B", "B", "B", "B"]);
 });
+
+/* eslint-disable no-sparse-arrays */
+/* eslint-disable comma-dangle */
+it("window computes a moving average", () => {
+  const data = range(6);
+  const m1 = Plot.windowX({k: 1, x: (d) => d, strict: true});
+  m1.transform(data, [range(data.length)]);
+  assert.deepStrictEqual(m1.x.transform(), [0, 1, 2, 3, 4, 5]);
+  const m2 = Plot.windowX({k: 2, x: (d) => d, strict: true});
+  m2.transform(data, [range(data.length)]);
+  assert.deepStrictEqual(m2.x.transform(), [0.5, 1.5, 2.5, 3.5, 4.5, ,]);
+  const m3 = Plot.windowX({k: 3, x: (d) => d, strict: true});
+  m3.transform(data, [range(data.length)]);
+  assert.deepStrictEqual(m3.x.transform(), [, 1, 2, 3, 4, ,]);
+  const m4 = Plot.windowX({k: 4, x: (d) => d, strict: true});
+  m4.transform(data, [range(data.length)]);
+  assert.deepStrictEqual(m4.x.transform(), [, 1.5, 2.5, 3.5, , ,]);
+});
+
+it("window skips NaN", () => {
+  const data = [1, 1, 1, null, 1, 1, 1, 1, 1, NaN, 1, 1, 1];
+  const m3 = Plot.windowX({k: 3, x: (d) => d, strict: true});
+  m3.transform(data, [range(data.length)]);
+  assert.deepStrictEqual(m3.x.transform(), [, 1, NaN, NaN, NaN, 1, 1, 1, NaN, NaN, NaN, 1, ,]);
+});
+
+it("window treats null as NaN", () => {
+  const data = [1, 1, 1, null, 1, 1, 1, 1, 1, null, 1, 1, 1];
+  const m3 = Plot.windowX({k: 3, x: (d) => d, strict: true});
+  m3.transform(data, [range(data.length)]);
+  assert.deepStrictEqual(m3.x.transform(), [, 1, NaN, NaN, NaN, 1, 1, 1, NaN, NaN, NaN, 1, ,]);
+});
+
+it("window respects anchor", () => {
+  const data = [0, 1, 2, 3, 4, 5];
+  const mc = Plot.windowX({k: 3, x: (d) => d, strict: true});
+  mc.transform(data, [range(data.length)]);
+  assert.deepStrictEqual(mc.x.transform(), [, 1, 2, 3, 4, ,]);
+  const ml = Plot.windowX({k: 3, anchor: "start", strict: true, x: (d) => d});
+  ml.transform(data, [range(data.length)]);
+  assert.deepStrictEqual(ml.x.transform(), [1, 2, 3, 4, , ,]);
+  const mt = Plot.windowX({k: 3, anchor: "end", strict: true, x: (d) => d});
+  mt.transform(data, [range(data.length)]);
+  assert.deepStrictEqual(mt.x.transform(), [, , 1, 2, 3, 4]);
+});
+
+it("window throws on non-numeric data", () => {
+  const data = [null, "A", 1, 2, 3, 4, 5];
+  const mc = Plot.windowX({k: 3, x: (d) => d});
+  assert.throws(() => mc.transform(data, [range(data.length)]));
+});