163 lines
22 KiB
JavaScript
163 lines
22 KiB
JavaScript
|
|
/**
|
||
|
|
* @license
|
||
|
|
* Copyright 2018 Google LLC. All Rights Reserved.
|
||
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||
|
|
* you may not use this file except in compliance with the License.
|
||
|
|
* You may obtain a copy of the License at
|
||
|
|
*
|
||
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||
|
|
*
|
||
|
|
* Unless required by applicable law or agreed to in writing, software
|
||
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||
|
|
* See the License for the specific language governing permissions and
|
||
|
|
* limitations under the License.
|
||
|
|
*
|
||
|
|
* =============================================================================
|
||
|
|
*/
|
||
|
|
import * as tf from '@tensorflow/tfjs-core';
|
||
|
|
import { div, max, min, sub } from '@tensorflow/tfjs-core';
|
||
|
|
/**
|
||
|
|
* Provides a function that scales numeric values into the [0, 1] interval.
|
||
|
|
*
|
||
|
|
* @param min the lower bound of the inputs, which should be mapped to 0.
|
||
|
|
* @param max the upper bound of the inputs, which should be mapped to 1,
|
||
|
|
* @return A function that maps an input ElementArray to a scaled ElementArray.
|
||
|
|
*/
|
||
|
|
export function scaleTo01(min, max) {
|
||
|
|
const range = max - min;
|
||
|
|
const minTensor = tf.scalar(min);
|
||
|
|
const rangeTensor = tf.scalar(range);
|
||
|
|
return (value) => {
|
||
|
|
if (typeof (value) === 'string') {
|
||
|
|
throw new Error('Can\'t scale a string.');
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
if (value instanceof tf.Tensor) {
|
||
|
|
const result = div(sub(value, minTensor), rangeTensor);
|
||
|
|
return result;
|
||
|
|
}
|
||
|
|
else if (value instanceof Array) {
|
||
|
|
return value.map(v => (v - min) / range);
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
return (value - min) / range;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
};
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Provides a function that calculates column level statistics, i.e. min, max,
|
||
|
|
* variance, stddev.
|
||
|
|
*
|
||
|
|
* @param dataset The Dataset object whose statistics will be calculated.
|
||
|
|
* @param sampleSize (Optional) If set, statistics will only be calculated
|
||
|
|
* against a subset of the whole data.
|
||
|
|
* @param shuffleWindowSize (Optional) If set, shuffle provided dataset before
|
||
|
|
* calculating statistics.
|
||
|
|
* @return A DatasetStatistics object that contains NumericColumnStatistics of
|
||
|
|
* each column.
|
||
|
|
*/
|
||
|
|
export async function computeDatasetStatistics(dataset, sampleSize, shuffleWindowSize) {
|
||
|
|
let sampleDataset = dataset;
|
||
|
|
// TODO(soergel): allow for deep shuffle where possible.
|
||
|
|
if (shuffleWindowSize != null) {
|
||
|
|
sampleDataset = sampleDataset.shuffle(shuffleWindowSize);
|
||
|
|
}
|
||
|
|
if (sampleSize != null) {
|
||
|
|
sampleDataset = sampleDataset.take(sampleSize);
|
||
|
|
}
|
||
|
|
// TODO(soergel): prepare the column objects based on a schema.
|
||
|
|
const result = {};
|
||
|
|
await sampleDataset.forEachAsync(e => {
|
||
|
|
for (const key of Object.keys(e)) {
|
||
|
|
const value = e[key];
|
||
|
|
if (typeof (value) === 'string') {
|
||
|
|
// No statistics for string element.
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
let previousMean = 0;
|
||
|
|
let previousLength = 0;
|
||
|
|
let previousVariance = 0;
|
||
|
|
let columnStats = result[key];
|
||
|
|
if (columnStats == null) {
|
||
|
|
columnStats = {
|
||
|
|
min: Number.POSITIVE_INFINITY,
|
||
|
|
max: Number.NEGATIVE_INFINITY,
|
||
|
|
mean: 0,
|
||
|
|
variance: 0,
|
||
|
|
stddev: 0,
|
||
|
|
length: 0
|
||
|
|
};
|
||
|
|
result[key] = columnStats;
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
previousMean = columnStats.mean;
|
||
|
|
previousLength = columnStats.length;
|
||
|
|
previousVariance = columnStats.variance;
|
||
|
|
}
|
||
|
|
let recordMin;
|
||
|
|
let recordMax;
|
||
|
|
// Calculate accumulated mean and variance following tf.Transform
|
||
|
|
// implementation
|
||
|
|
let valueLength = 0;
|
||
|
|
let valueMean = 0;
|
||
|
|
let valueVariance = 0;
|
||
|
|
let combinedLength = 0;
|
||
|
|
let combinedMean = 0;
|
||
|
|
let combinedVariance = 0;
|
||
|
|
if (value instanceof tf.Tensor) {
|
||
|
|
recordMin = min(value).dataSync()[0];
|
||
|
|
recordMax = max(value).dataSync()[0];
|
||
|
|
const valueMoment = tf.moments(value);
|
||
|
|
valueMean = valueMoment.mean.dataSync()[0];
|
||
|
|
valueVariance = valueMoment.variance.dataSync()[0];
|
||
|
|
valueLength = value.size;
|
||
|
|
}
|
||
|
|
else if (value instanceof Array) {
|
||
|
|
recordMin = value.reduce((a, b) => Math.min(a, b));
|
||
|
|
recordMax = value.reduce((a, b) => Math.max(a, b));
|
||
|
|
const valueMoment = tf.moments(value);
|
||
|
|
valueMean = valueMoment.mean.dataSync()[0];
|
||
|
|
valueVariance = valueMoment.variance.dataSync()[0];
|
||
|
|
valueLength = value.length;
|
||
|
|
}
|
||
|
|
else if (!isNaN(value) && isFinite(value)) {
|
||
|
|
recordMin = value;
|
||
|
|
recordMax = value;
|
||
|
|
valueMean = value;
|
||
|
|
valueVariance = 0;
|
||
|
|
valueLength = 1;
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
columnStats = null;
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
combinedLength = previousLength + valueLength;
|
||
|
|
combinedMean = previousMean +
|
||
|
|
(valueLength / combinedLength) * (valueMean - previousMean);
|
||
|
|
combinedVariance = previousVariance +
|
||
|
|
(valueLength / combinedLength) *
|
||
|
|
(valueVariance +
|
||
|
|
((valueMean - combinedMean) * (valueMean - previousMean)) -
|
||
|
|
previousVariance);
|
||
|
|
columnStats.min = Math.min(columnStats.min, recordMin);
|
||
|
|
columnStats.max = Math.max(columnStats.max, recordMax);
|
||
|
|
columnStats.length = combinedLength;
|
||
|
|
columnStats.mean = combinedMean;
|
||
|
|
columnStats.variance = combinedVariance;
|
||
|
|
columnStats.stddev = Math.sqrt(combinedVariance);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
});
|
||
|
|
// Variance and stddev should be NaN for the case of a single element.
|
||
|
|
for (const key in result) {
|
||
|
|
const stat = result[key];
|
||
|
|
if (stat.length === 1) {
|
||
|
|
stat.variance = NaN;
|
||
|
|
stat.stddev = NaN;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return result;
|
||
|
|
}
|
||
|
|
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic3RhdGlzdGljcy5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uLy4uLy4uL3RmanMtZGF0YS9zcmMvc3RhdGlzdGljcy50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQTs7Ozs7Ozs7Ozs7Ozs7OztHQWdCRztBQUVILE9BQU8sS0FBSyxFQUFFLE1BQU0sdUJBQXVCLENBQUM7QUFDNUMsT0FBTyxFQUFDLEdBQUcsRUFBRSxHQUFHLEVBQUUsR0FBRyxFQUFFLEdBQUcsRUFBQyxNQUFNLHVCQUF1QixDQUFDO0FBOEN6RDs7Ozs7O0dBTUc7QUFDSCxNQUFNLFVBQVUsU0FBUyxDQUFDLEdBQVcsRUFBRSxHQUFXO0lBRWhELE1BQU0sS0FBSyxHQUFHLEdBQUcsR0FBRyxHQUFHLENBQUM7SUFDeEIsTUFBTSxTQUFTLEdBQWMsRUFBRSxDQUFDLE1BQU0sQ0FBQyxHQUFHLENBQUMsQ0FBQztJQUM1QyxNQUFNLFdBQVcsR0FBYyxFQUFFLENBQUMsTUFBTSxDQUFDLEtBQUssQ0FBQyxDQUFDO0lBQ2hELE9BQU8sQ0FBQyxLQUFtQixFQUFnQixFQUFFO1FBQzNDLElBQUksT0FBTyxDQUFDLEtBQUssQ0FBQyxLQUFLLFFBQVEsRUFBRTtZQUMvQixNQUFNLElBQUksS0FBSyxDQUFDLHdCQUF3QixDQUFDLENBQUM7U0FDM0M7YUFBTTtZQUNMLElBQUksS0FBSyxZQUFZLEVBQUUsQ0FBQyxNQUFNLEVBQUU7Z0JBQzlCLE1BQU0sTUFBTSxHQUFHLEdBQUcsQ0FBQyxHQUFHLENBQUMsS0FBSyxFQUFFLFNBQVMsQ0FBQyxFQUFFLFdBQVcsQ0FBQyxDQUFDO2dCQUN2RCxPQUFPLE1BQU0sQ0FBQzthQUNmO2lCQUFNLElBQUksS0FBSyxZQUFZLEtBQUssRUFBRTtnQkFDakMsT0FBTyxLQUFLLENBQUMsR0FBRyxDQUFDLENBQUMsQ0FBQyxFQUFFLENBQUMsQ0FBQyxDQUFDLEdBQUcsR0FBRyxDQUFDLEdBQUcsS0FBSyxDQUFDLENBQUM7YUFDMUM7aUJBQU07Z0JBQ0wsT0FBTyxDQUFDLEtBQUssR0FBRyxHQUFHLENBQUMsR0FBRyxLQUFLLENBQUM7YUFDOUI7U0FDRjtJQUNILENBQUMsQ0FBQztBQUNKLENBQUM7QUFFRDs7Ozs7Ozs7Ozs7R0FXRztBQUNILE1BQU0sQ0FBQyxLQUFLLFVBQVUsd0JBQXdCLENBQzFDLE9BQStCLEVBQUUsVUFBbUIsRUFDcEQsaUJBQTBCO0lBQzVCLElBQUksYUFBYSxHQUFHLE9BQU8sQ0FBQztJQUM1Qix3REFBd0Q7SUFDeEQsSUFBSSxpQkFBaUIsSUFBSSxJQUFJLEVBQUU7UUFDN0IsYUFBYSxHQUFHLGFBQWEsQ0FBQyxPQUFPLENBQUMsaUJBQWlCLENBQUMsQ0FBQztLQUMxRDtJQUNELElBQUksVUFBVSxJQUFJLElBQUksRUFBRTtRQUN0QixhQUFhLEdBQUcsYUFBYSxDQUFDLElBQUksQ0FBQyxVQUFVLENBQUMsQ0FBQztLQUNoRDtJQUVELCtEQUErRDtJQUMvRCxNQUFNLE1BQU0sR0FBc0IsRUFBRSxDQUFDO0lBRXJDLE1BQU0sYUFBYSxDQUFDLFlBQVksQ0FBQyxDQUFDLENBQUMsRUFBRTtRQUNuQyxLQUFLLE1BQU0sR0FBRyxJQUFJLE1BQU0sQ0FBQyxJQUFJLENBQUMsQ0FBQyxDQUFDLEVBQUU7WUFDaEMsTUFBTSxLQUFLLEdBQUcsQ0FBQyxDQUFDLEdBQUcsQ0FBQyxDQUFDO1lBQ3JCLElBQUksT0FBTyxDQUFDLEtBQUssQ0FBQyxLQUFLLFFBQVEsRUFBRTtnQkFDL0Isb0NBQW9DO2FBQ3JDO2lCQUFNO2dCQUNMLElBQUksWUFBWSxHQUFHLENBQUMsQ0FBQztnQkFDckIsSUFBSSxjQUFjLEdBQUcsQ0FBQyxDQUFDO2dCQUN2QixJQUFJLGdCQUFnQixHQUFHLENBQUMsQ0FBQztnQkFDekIsSUFBSSxXQUFXLEdBQTRCLE1BQU0sQ0FBQyxHQUFHLENBQUMsQ0FBQztnQkFDdkQsSUFBSSxXQUFXLElBQUksSUFBSSxFQUFFO29CQUN2QixXQUFXLEdBQUc7d0JBQ1osR0FBRyxFQUFFLE1BQU0sQ0FBQyxpQkFBaUI7d0JBQzdCLEdBQUcsRUFBRSxNQUFNLENBQUMsaUJBQWlCO3dCQUM3QixJQUFJLEVBQUUsQ0FBQzt3QkFDUCxRQUFRLEVBQUUsQ0FBQzt3QkFDWCxNQUFNLEVBQUUsQ0FBQzt3QkFDVCxNQUFNLEVBQUUsQ0FBQztxQkFDVixDQUFDO29CQUNGLE1BQU0sQ0FBQyxHQUFHLENBQUMsR0FBRyxXQUFXLENBQUM7aUJBQzNCO3FCQUFNO29CQUNMLFlBQVksR0FBRyxXQUFXLENBQUMsSUFBSSxDQUFDO29CQUNoQyxjQUFjLEdBQUcsV0FBVyxDQUFDLE1BQU0sQ0FBQztvQkFDcEMsZ0JBQWdCLEdBQUcsV0FBVyxDQUFDLFFBQVEsQ0FBQztpQkFDekM7Z0JBQ0QsSUFBSSxTQUFpQixDQUFDO2dCQUN0QixJQUFJLFNBQWlCLENBQUM7Z0JBRXRCLGlFQUFpRTtnQkFDakUsaUJBQWlCO2dCQUNqQixJQUFJLFdBQVcsR0FBRyxDQUFDLENBQUM7Z0JBQ3BCLElBQUksU0FBUyxHQUFHLENBQUMsQ0FBQztnQkFDbEIsSUFBSSxhQUFhLEdBQUcsQ0FBQyxDQUFDO2dCQUN0QixJQUFJLGNBQWMsR0FBRyxDQUFDLENBQUM7Z0JBQ3ZCLElBQUksWUFBWSxHQUFHLENBQUMsQ0FBQztnQkFDckIsSUFBSSxnQkFBZ0IsR0FBRyxDQUFDLENBQUM7Z0JBRXpCLElBQUksS0FBSyxZQUFZLEVBQUUsQ0FBQyxNQUFNLEVBQUU7b0JBQzlCLFNBQVMsR0FBRyxHQUFHLENBQUMsS0FBSyxDQUFDLENBQUMsUUFBUSxFQUFFLENBQUMsQ0FBQyxDQUFDLENBQUM7b0JBQ3JDLFNBQVMsR0FBRyxHQUFHLENBQUMsS0FBSyxDQUFDLENBQUMsUUFBUSxFQUFFLENBQUMsQ0FBQyxDQUFDLENBQUM7b0JBQ3JDLE1BQU0sV0FBVyxHQUFHLEVBQUUsQ0FBQyxPQUFPLENBQUMsS0FBSyxDQUFDLENBQUM7b0JBQ3RDLFNBQVMsR0FBRyxXQUFXLENBQUMsSUFBSSxDQUFDLFFBQVEsRUFBRSxDQUFDLENBQUMsQ0FBQyxDQUFDO29CQUMzQyxhQUFhLEdBQUcsV0FBVyxDQUFDLFFBQVEsQ0FBQyxRQUFRLEVBQUUsQ0FBQyxDQUFDLENBQUMsQ0FBQztvQkFDbkQsV0FBVyxHQUFHLEtBQUssQ0FBQyxJQUFJLENBQUM7aUJBRTFCO3FCQUFNLElBQUksS0FBSyxZQUFZLEtBQUssRUFBRTtvQkFDakMsU0FBUyxHQUFHLEtBQUssQ0FBQyxNQUFNLENBQUMsQ0FBQyxDQUFDLEVBQUUsQ0FBQyxFQUFFLEVBQUUsQ0FBQyxJQUFJLENBQUMsR0FBRyxDQUFDLENBQUMsRUFBRSxDQUFDLENBQUMsQ0FBQyxDQUFDO29CQUNuRCxTQUFTLEdBQUcsS0FBSyxDQUFDLE1BQU0sQ0FBQyxDQUFDLENBQUMsRUFBRSxDQUFDLEVBQUUsRUFBRSxDQUFDLElBQUksQ0FBQyxHQUFHLENBQUMsQ0FBQ
|