163 lines
22 KiB
JavaScript
Raw Permalink Normal View History

2024-10-15 23:05:53 +03:00
/**
* @license
* Copyright 2018 Google LLC. All Rights Reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* =============================================================================
*/
import * as tf from '@tensorflow/tfjs-core';
import { div, max, min, sub } from '@tensorflow/tfjs-core';
/**
* Provides a function that scales numeric values into the [0, 1] interval.
*
* @param min the lower bound of the inputs, which should be mapped to 0.
* @param max the upper bound of the inputs, which should be mapped to 1,
* @return A function that maps an input ElementArray to a scaled ElementArray.
*/
export function scaleTo01(min, max) {
const range = max - min;
const minTensor = tf.scalar(min);
const rangeTensor = tf.scalar(range);
return (value) => {
if (typeof (value) === 'string') {
throw new Error('Can\'t scale a string.');
}
else {
if (value instanceof tf.Tensor) {
const result = div(sub(value, minTensor), rangeTensor);
return result;
}
else if (value instanceof Array) {
return value.map(v => (v - min) / range);
}
else {
return (value - min) / range;
}
}
};
}
/**
* Provides a function that calculates column level statistics, i.e. min, max,
* variance, stddev.
*
* @param dataset The Dataset object whose statistics will be calculated.
* @param sampleSize (Optional) If set, statistics will only be calculated
* against a subset of the whole data.
* @param shuffleWindowSize (Optional) If set, shuffle provided dataset before
* calculating statistics.
* @return A DatasetStatistics object that contains NumericColumnStatistics of
* each column.
*/
export async function computeDatasetStatistics(dataset, sampleSize, shuffleWindowSize) {
let sampleDataset = dataset;
// TODO(soergel): allow for deep shuffle where possible.
if (shuffleWindowSize != null) {
sampleDataset = sampleDataset.shuffle(shuffleWindowSize);
}
if (sampleSize != null) {
sampleDataset = sampleDataset.take(sampleSize);
}
// TODO(soergel): prepare the column objects based on a schema.
const result = {};
await sampleDataset.forEachAsync(e => {
for (const key of Object.keys(e)) {
const value = e[key];
if (typeof (value) === 'string') {
// No statistics for string element.
}
else {
let previousMean = 0;
let previousLength = 0;
let previousVariance = 0;
let columnStats = result[key];
if (columnStats == null) {
columnStats = {
min: Number.POSITIVE_INFINITY,
max: Number.NEGATIVE_INFINITY,
mean: 0,
variance: 0,
stddev: 0,
length: 0
};
result[key] = columnStats;
}
else {
previousMean = columnStats.mean;
previousLength = columnStats.length;
previousVariance = columnStats.variance;
}
let recordMin;
let recordMax;
// Calculate accumulated mean and variance following tf.Transform
// implementation
let valueLength = 0;
let valueMean = 0;
let valueVariance = 0;
let combinedLength = 0;
let combinedMean = 0;
let combinedVariance = 0;
if (value instanceof tf.Tensor) {
recordMin = min(value).dataSync()[0];
recordMax = max(value).dataSync()[0];
const valueMoment = tf.moments(value);
valueMean = valueMoment.mean.dataSync()[0];
valueVariance = valueMoment.variance.dataSync()[0];
valueLength = value.size;
}
else if (value instanceof Array) {
recordMin = value.reduce((a, b) => Math.min(a, b));
recordMax = value.reduce((a, b) => Math.max(a, b));
const valueMoment = tf.moments(value);
valueMean = valueMoment.mean.dataSync()[0];
valueVariance = valueMoment.variance.dataSync()[0];
valueLength = value.length;
}
else if (!isNaN(value) && isFinite(value)) {
recordMin = value;
recordMax = value;
valueMean = value;
valueVariance = 0;
valueLength = 1;
}
else {
columnStats = null;
continue;
}
combinedLength = previousLength + valueLength;
combinedMean = previousMean +
(valueLength / combinedLength) * (valueMean - previousMean);
combinedVariance = previousVariance +
(valueLength / combinedLength) *
(valueVariance +
((valueMean - combinedMean) * (valueMean - previousMean)) -
previousVariance);
columnStats.min = Math.min(columnStats.min, recordMin);
columnStats.max = Math.max(columnStats.max, recordMax);
columnStats.length = combinedLength;
columnStats.mean = combinedMean;
columnStats.variance = combinedVariance;
columnStats.stddev = Math.sqrt(combinedVariance);
}
}
});
// Variance and stddev should be NaN for the case of a single element.
for (const key in result) {
const stat = result[key];
if (stat.length === 1) {
stat.variance = NaN;
stat.stddev = NaN;
}
}
return result;
}
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic3RhdGlzdGljcy5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uLy4uLy4uLy4uLy4uL3RmanMtZGF0YS9zcmMvc3RhdGlzdGljcy50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQTs7Ozs7Ozs7Ozs7Ozs7OztHQWdCRztBQUVILE9BQU8sS0FBSyxFQUFFLE1BQU0sdUJBQXVCLENBQUM7QUFDNUMsT0FBTyxFQUFDLEdBQUcsRUFBRSxHQUFHLEVBQUUsR0FBRyxFQUFFLEdBQUcsRUFBQyxNQUFNLHVCQUF1QixDQUFDO0FBOEN6RDs7Ozs7O0dBTUc7QUFDSCxNQUFNLFVBQVUsU0FBUyxDQUFDLEdBQVcsRUFBRSxHQUFXO0lBRWhELE1BQU0sS0FBSyxHQUFHLEdBQUcsR0FBRyxHQUFHLENBQUM7SUFDeEIsTUFBTSxTQUFTLEdBQWMsRUFBRSxDQUFDLE1BQU0sQ0FBQyxHQUFHLENBQUMsQ0FBQztJQUM1QyxNQUFNLFdBQVcsR0FBYyxFQUFFLENBQUMsTUFBTSxDQUFDLEtBQUssQ0FBQyxDQUFDO0lBQ2hELE9BQU8sQ0FBQyxLQUFtQixFQUFnQixFQUFFO1FBQzNDLElBQUksT0FBTyxDQUFDLEtBQUssQ0FBQyxLQUFLLFFBQVEsRUFBRTtZQUMvQixNQUFNLElBQUksS0FBSyxDQUFDLHdCQUF3QixDQUFDLENBQUM7U0FDM0M7YUFBTTtZQUNMLElBQUksS0FBSyxZQUFZLEVBQUUsQ0FBQyxNQUFNLEVBQUU7Z0JBQzlCLE1BQU0sTUFBTSxHQUFHLEdBQUcsQ0FBQyxHQUFHLENBQUMsS0FBSyxFQUFFLFNBQVMsQ0FBQyxFQUFFLFdBQVcsQ0FBQyxDQUFDO2dCQUN2RCxPQUFPLE1BQU0sQ0FBQzthQUNmO2lCQUFNLElBQUksS0FBSyxZQUFZLEtBQUssRUFBRTtnQkFDakMsT0FBTyxLQUFLLENBQUMsR0FBRyxDQUFDLENBQUMsQ0FBQyxFQUFFLENBQUMsQ0FBQyxDQUFDLEdBQUcsR0FBRyxDQUFDLEdBQUcsS0FBSyxDQUFDLENBQUM7YUFDMUM7aUJBQU07Z0JBQ0wsT0FBTyxDQUFDLEtBQUssR0FBRyxHQUFHLENBQUMsR0FBRyxLQUFLLENBQUM7YUFDOUI7U0FDRjtJQUNILENBQUMsQ0FBQztBQUNKLENBQUM7QUFFRDs7Ozs7Ozs7Ozs7R0FXRztBQUNILE1BQU0sQ0FBQyxLQUFLLFVBQVUsd0JBQXdCLENBQzFDLE9BQStCLEVBQUUsVUFBbUIsRUFDcEQsaUJBQTBCO0lBQzVCLElBQUksYUFBYSxHQUFHLE9BQU8sQ0FBQztJQUM1Qix3REFBd0Q7SUFDeEQsSUFBSSxpQkFBaUIsSUFBSSxJQUFJLEVBQUU7UUFDN0IsYUFBYSxHQUFHLGFBQWEsQ0FBQyxPQUFPLENBQUMsaUJBQWlCLENBQUMsQ0FBQztLQUMxRDtJQUNELElBQUksVUFBVSxJQUFJLElBQUksRUFBRTtRQUN0QixhQUFhLEdBQUcsYUFBYSxDQUFDLElBQUksQ0FBQyxVQUFVLENBQUMsQ0FBQztLQUNoRDtJQUVELCtEQUErRDtJQUMvRCxNQUFNLE1BQU0sR0FBc0IsRUFBRSxDQUFDO0lBRXJDLE1BQU0sYUFBYSxDQUFDLFlBQVksQ0FBQyxDQUFDLENBQUMsRUFBRTtRQUNuQyxLQUFLLE1BQU0sR0FBRyxJQUFJLE1BQU0sQ0FBQyxJQUFJLENBQUMsQ0FBQyxDQUFDLEVBQUU7WUFDaEMsTUFBTSxLQUFLLEdBQUcsQ0FBQyxDQUFDLEdBQUcsQ0FBQyxDQUFDO1lBQ3JCLElBQUksT0FBTyxDQUFDLEtBQUssQ0FBQyxLQUFLLFFBQVEsRUFBRTtnQkFDL0Isb0NBQW9DO2FBQ3JDO2lCQUFNO2dCQUNMLElBQUksWUFBWSxHQUFHLENBQUMsQ0FBQztnQkFDckIsSUFBSSxjQUFjLEdBQUcsQ0FBQyxDQUFDO2dCQUN2QixJQUFJLGdCQUFnQixHQUFHLENBQUMsQ0FBQztnQkFDekIsSUFBSSxXQUFXLEdBQTRCLE1BQU0sQ0FBQyxHQUFHLENBQUMsQ0FBQztnQkFDdkQsSUFBSSxXQUFXLElBQUksSUFBSSxFQUFFO29CQUN2QixXQUFXLEdBQUc7d0JBQ1osR0FBRyxFQUFFLE1BQU0sQ0FBQyxpQkFBaUI7d0JBQzdCLEdBQUcsRUFBRSxNQUFNLENBQUMsaUJBQWlCO3dCQUM3QixJQUFJLEVBQUUsQ0FBQzt3QkFDUCxRQUFRLEVBQUUsQ0FBQzt3QkFDWCxNQUFNLEVBQUUsQ0FBQzt3QkFDVCxNQUFNLEVBQUUsQ0FBQztxQkFDVixDQUFDO29CQUNGLE1BQU0sQ0FBQyxHQUFHLENBQUMsR0FBRyxXQUFXLENBQUM7aUJBQzNCO3FCQUFNO29CQUNMLFlBQVksR0FBRyxXQUFXLENBQUMsSUFBSSxDQUFDO29CQUNoQyxjQUFjLEdBQUcsV0FBVyxDQUFDLE1BQU0sQ0FBQztvQkFDcEMsZ0JBQWdCLEdBQUcsV0FBVyxDQUFDLFFBQVEsQ0FBQztpQkFDekM7Z0JBQ0QsSUFBSSxTQUFpQixDQUFDO2dCQUN0QixJQUFJLFNBQWlCLENBQUM7Z0JBRXRCLGlFQUFpRTtnQkFDakUsaUJBQWlCO2dCQUNqQixJQUFJLFdBQVcsR0FBRyxDQUFDLENBQUM7Z0JBQ3BCLElBQUksU0FBUyxHQUFHLENBQUMsQ0FBQztnQkFDbEIsSUFBSSxhQUFhLEdBQUcsQ0FBQyxDQUFDO2dCQUN0QixJQUFJLGNBQWMsR0FBRyxDQUFDLENBQUM7Z0JBQ3ZCLElBQUksWUFBWSxHQUFHLENBQUMsQ0FBQztnQkFDckIsSUFBSSxnQkFBZ0IsR0FBRyxDQUFDLENBQUM7Z0JBRXpCLElBQUksS0FBSyxZQUFZLEVBQUUsQ0FBQyxNQUFNLEVBQUU7b0JBQzlCLFNBQVMsR0FBRyxHQUFHLENBQUMsS0FBSyxDQUFDLENBQUMsUUFBUSxFQUFFLENBQUMsQ0FBQyxDQUFDLENBQUM7b0JBQ3JDLFNBQVMsR0FBRyxHQUFHLENBQUMsS0FBSyxDQUFDLENBQUMsUUFBUSxFQUFFLENBQUMsQ0FBQyxDQUFDLENBQUM7b0JBQ3JDLE1BQU0sV0FBVyxHQUFHLEVBQUUsQ0FBQyxPQUFPLENBQUMsS0FBSyxDQUFDLENBQUM7b0JBQ3RDLFNBQVMsR0FBRyxXQUFXLENBQUMsSUFBSSxDQUFDLFFBQVEsRUFBRSxDQUFDLENBQUMsQ0FBQyxDQUFDO29CQUMzQyxhQUFhLEdBQUcsV0FBVyxDQUFDLFFBQVEsQ0FBQyxRQUFRLEVBQUUsQ0FBQyxDQUFDLENBQUMsQ0FBQztvQkFDbkQsV0FBVyxHQUFHLEtBQUssQ0FBQyxJQUFJLENBQUM7aUJBRTFCO3FCQUFNLElBQUksS0FBSyxZQUFZLEtBQUssRUFBRTtvQkFDakMsU0FBUyxHQUFHLEtBQUssQ0FBQyxNQUFNLENBQUMsQ0FBQyxDQUFDLEVBQUUsQ0FBQyxFQUFFLEVBQUUsQ0FBQyxJQUFJLENBQUMsR0FBRyxDQUFDLENBQUMsRUFBRSxDQUFDLENBQUMsQ0FBQyxDQUFDO29CQUNuRCxTQUFTLEdBQUcsS0FBSyxDQUFDLE1BQU0sQ0FBQyxDQUFDLENBQUMsRUFBRSxDQUFDLEVBQUUsRUFBRSxDQUFDLElBQUksQ0FBQyxHQUFHLENBQUMsQ0FBQ