From 432a9f8ca9096b110ed1e15355e2669dd5d1fa00 Mon Sep 17 00:00:00 2001 From: Steve Morley Date: Wed, 26 Sep 2018 18:28:46 -0600 Subject: [PATCH 1/4] verify/__init__.py: expanded docstrings Also a minor reordering of a conditional block for speed and a setting for mask behaviour, both in medSymAccuracy --- verify/__init__.py | 323 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 279 insertions(+), 44 deletions(-) diff --git a/verify/__init__.py b/verify/__init__.py index f165a50..81dba6d 100644 --- a/verify/__init__.py +++ b/verify/__init__.py @@ -1,5 +1,10 @@ '''Module containing verification and performance metrics +With the exception of the ContingencyNxN and Contingency2x2 classes, +the inputs for all metrics are assumed to be array-like and 1D. Bad +values are assumed to be stored as NaN and these are excluded in +metric calculations. + Author: Steve Morley Institution: Los Alamos National Laboratory Contact: smorley@lanl.gov @@ -33,12 +38,12 @@ def skill(A_data, A_ref, A_perf=0): Accuracy measure of data set A_ref : float Accuracy measure for reference forecast - A_perf : float + A_perf : float, optional Accuracy measure for "perfect forecast" (Default = 0) Returns ======= - out : float + ss_ref : float Forecast skill for the given forecast, relative to the reference, using the chosen accuracy measure ''' @@ -55,6 +60,21 @@ def percBetter(predict1, predict2, observed): For example, if we want to know whether a new forecast performs better than a reference forecast... + Parameters + ========== + predict1 : array-like + Array-like (list, numpy array, etc.) of predictions from model A + predict2 : array-like + Array-like (list, numpy array, etc.) of predictions from model B + observed : array-like + Array-like (list, numpy array, etc.) of observed values of scalar quantity + + Returns + ======= + percBetter : float + The percentage of observations where method A was closer to observation than method B + + Examples ======== >>> import verify @@ -86,6 +106,18 @@ def percBetter(predict1, predict2, observed): def bias(predicted, observed): ''' Scale-dependent bias as measured by the mean error + + Parameters + ========== + predicted : array-like + Array-like (list, numpy array, etc.) of predictions + observed : array-like + Array-like (list, numpy array, etc.) of observed values of scalar quantity + + Returns + ======= + bias : float + Mean error of prediction ''' pred = _maskSeries(predicted) obse = _maskSeries(observed) @@ -95,6 +127,18 @@ def bias(predicted, observed): def meanPercentageError(predicted, observed): ''' Order-dependent bias as measured by the mean percentage error + + Parameters + ========== + predicted : array-like + Array-like (list, numpy array, etc.) of predictions + observed : array-like + Array-like (list, numpy array, etc.) of observed values of scalar quantity + + Returns + ======= + mpe : float + Mean percentage error of prediction ''' pred = _maskSeries(predicted) obse = _maskSeries(observed) @@ -105,6 +149,26 @@ def meanPercentageError(predicted, observed): def medianLogAccuracy(predicted, observed, mfunc=np.median, base=10): ''' Order-dependent bias as measured by the median of the log accuracy ratio + + Parameters + ========== + predicted : array-like + Array-like (list, numpy array, etc.) of predictions + observed : array-like + Array-like (list, numpy array, etc.) of observed values of scalar quantity + + Other Parameters + ================ + mfunc : function, optional + Function to use for central tendency (default: numpy.median) + base : number, optional + Base to use for logarithmic transform (default: 10) + + Returns + ======= + mla : float + Median log accuracy of prediction + ''' pred = _maskSeries(predicted) obse = _maskSeries(observed) @@ -130,7 +194,25 @@ def symmetricSignedBias(predicted, observed): #======= Accuracy measures =======# def accuracy(data, climate=None): - ''' + '''Convenience function to calculate a selection of unscaled accuracy measures + + Parameters + ========== + data : array-like + Array-like (list, numpy array, etc.) of predictions + climate : array-like or float, optional + Array-like (list, numpy array, etc.) or float of observed values of scalar quantity. + If climate is None (default) then the accuracy is assessed relative to persistence. + + Returns + ======= + out : dict + Dictionary containing unscaled accuracy measures + + See Also + ======== + meanSquaredError, RMSE, meanAbsError, medAbsError + ''' data = _maskSeries(data) if climate is not None: @@ -153,11 +235,9 @@ def meanSquaredError(data, climate=None): ========== data : array-like data to calculate mean squared error, default reference is persistence - - Other Parameters - ================ - climate: float - climatological mean to use as reference value + climate : array-like or float, optional + Array-like (list, numpy array, etc.) or float of observed values of scalar quantity. + If climate is None (default) then the accuracy is assessed relative to persistence. Returns ======= @@ -180,19 +260,17 @@ def meanSquaredError(data, climate=None): def RMSE(data, climate=None): '''Calculate the root mean squared error of a data set relative to some reference value - The chosen reference can be persistence, a provided climatological mean (scalar) - or a provided climatology (observation vector). + The chosen reference can be persistence (climate=None), a provided climatological + mean (scalar) or a provided climatology (observation vector). Parameters ========== data : array-like data to calculate mean squared error, default reference is persistence + climate : array-like or float, optional + Array-like (list, numpy array, etc.) or float of observed values of scalar quantity. + If climate is None (default) then the accuracy is assessed relative to persistence. - Other Parameters - ================ - climate: float - climatological mean to use as reference value - Returns ======= out : float @@ -218,12 +296,10 @@ def meanAbsError(data, climate=None): ========== data : array-like data to calculate mean squared error, default reference is persistence + climate : array-like or float, optional + Array-like (list, numpy array, etc.) or float of observed values of scalar quantity. + If climate is None (default) then the accuracy is assessed relative to persistence. - Other Parameters - ================ - climate: float - climatology to use as reference - Returns ======= out : float @@ -250,12 +326,10 @@ def medAbsError(data, climate=None): ========== data : array-like data to calculate median absolute error, default reference is persistence + climate : array-like or float, optional + Array-like (list, numpy array, etc.) or float of observed values of scalar quantity. + If climate is None (default) then the accuracy is assessed relative to persistence. - Other Parameters - ================ - climate: float - climatology to use as reference - Returns ======= out : float @@ -276,6 +350,23 @@ def medAbsError(data, climate=None): #======= Scaled/Relative Accuracy measures =======# def scaledAccuracy(predicted, observed): ''' + Convenience function to calculate a selection of relative, or scaled, accuracy measures + + Parameters + ========== + predicted : array-like + Array-like (list, numpy array, etc.) of predictions + observed : array-like + Array-like (list, numpy array, etc.) of observed values of scalar quantity + + Returns + ======= + out : dict + Dictionary containing scaled or relative accuracy measures + + See Also + ======== + medSymAccuracy, meanAPE, MASE, nRMSE ''' metrics = {'nRMSE': nRMSE, 'MASE': MASE, 'MAPE': meanAPE, 'MdAPE': functools.partial(meanAPE, mfunc=np.median), @@ -303,7 +394,7 @@ def nRMSE(predicted, observed): predicted: array-like predicted data for which to calculate mean squared error observed: float - observation vector (or climatological value (scalar)) to use as reference value + observation vector (or climatological value (scalar) to use as reference value) Returns ======= @@ -336,6 +427,22 @@ def scaledError(predicted, observed): References: R.J. Hyndman and A.B. Koehler, Another look at measures of forecast accuracy, Intl. J. Forecasting, 22, pp. 679-688, 2006. + + Parameters + ========== + predicted: array-like + predicted data for which to calculate scaled error + observed: float + observation vector (or climatological value (scalar) to use as reference value) + + Returns + ======= + out : float + the scaled error of the data set + + See Also + ======== + MASE ''' n_pts = len(predicted.ravel()) try: @@ -355,7 +462,29 @@ def scaledError(predicted, observed): def MASE(predicted, observed): - '''Mean Absolute Scaled Error''' + '''Mean Absolute Scaled Error + + References: + R.J. Hyndman and A.B. Koehler, Another look at measures of forecast + accuracy, Intl. J. Forecasting, 22, pp. 679-688, 2006. + + Parameters + ========== + predicted: array-like + predicted data for which to calculate MASE + observed: float + observation vector (or climatological value (scalar) to use as reference value) + + Returns + ======= + out : float + the mean absolute scaled error of the data set + + See Also + ======== + scaledError + + ''' q = scaledError(predicted, observed) n_pts = len(predicted.ravel()) return np.abs(q).sum()/n_pts @@ -363,6 +492,32 @@ def MASE(predicted, observed): def forecastError(predicted, observed, full=True): '''Error, defined using the sign convention of Jolliffe and Stephenson (Ch. 5) + + Parameters + ========== + predicted : array-like + Array-like (list, numpy array, etc.) of predictions + observed : array-like + Array-like (list, numpy array, etc.) of observed values of scalar quantity + + Other Parameters + ================ + full : boolean, optional + Switch determining nature of return value. When it is True (the + default) the function returns the errors as well as the predicted + and observed values as numpy arrays of floats, when False only the + array of forecast errors is returned. + + Returns + ======= + err : array + Array of forecast errors + + pred : array + Present only if `full` = True. Array of predicted values as floats + + obse : array + Present only if `full` = True. Array of observed values as floats ''' pred = np.asanyarray(predicted).astype(float) obse = np.asanyarray(observed).astype(float) @@ -376,6 +531,17 @@ def forecastError(predicted, observed, full=True): def percError(predicted, observed): '''Percentage Error + Parameters + ========== + predicted : array-like + Array-like (list, numpy array, etc.) of predictions + observed : array-like + Array-like (list, numpy array, etc.) of observed values of scalar quantity + + Returns + ======= + perc : array + Array of forecast errors expressed as a percentage ''' err, pred, obse = forecastError(predicted, observed, full=True) res = err/obse @@ -384,30 +550,78 @@ def percError(predicted, observed): def absPercError(predicted, observed): '''Absolute percentage error + + Parameters + ========== + predicted : array-like + Array-like (list, numpy array, etc.) of predictions + observed : array-like + Array-like (list, numpy array, etc.) of observed values of scalar quantity + + Returns + ======= + perc : array + Array of absolute percentage errors ''' err, pred, obse = forecastError(predicted, observed, full=True) res = np.abs(err/obse) return 100*res -def logAccuracy(predicted, observed, base=10): +def logAccuracy(predicted, observed, base=10, mask=True): '''Log Accuracy Ratio, defined as log(predicted/observed) or log(predicted)-log(observed) Using base 2 is computationally much faster, so unless the base is important to interpretation we recommend using that. + + Parameters + ========== + predicted : array-like + Array-like (list, numpy array, etc.) of predictions + observed : array-like + Array-like (list, numpy array, etc.) of observed values of scalar quantity + + Other Parameters + ================ + base : number, optional + Base to use for logarithmic transform (default: 10) + mask : boolean, optional + Switch to set masking behaviour. If True (default) the function will mask out NaN and negative values, + and will return a masked array. If False, the presence of negative numbers will raise a ValueError and + NaN will propagate through the calculation. + + Returns + ======= + logacc : array or masked array + Array of absolute percentage errors ''' - pred = _maskSeries(predicted) - obse = _maskSeries(observed) + if mask: + pred = _maskSeries(predicted) + obse = _maskSeries(observed) + negs_p = predicted <= 0 + negs_o = observed <= 0 + pred.mask = np.logical_or(pred.mask, negs_p) + obse.mask = np.logical_or(obse.mask, negs_o) + else: + pred = np.asanyarray(predicted) + obse = np.asanyarray(observed) #check for positivity - if (pred<=0).any() and (obse<=0).any(): + if (pred<=0).any() or (obse<=0).any(): raise ValueError('logAccuracy: input data are required to be positive') logfuncs = {10: np.log10, 2: np.log2, 'e': np.log} if base not in logfuncs: - raise NotImplementedError - return logfuncs[base](pred/obse) + supportedbases = '['+ ', '.join([str(key) for key in logfuncs]) + ']' + raise NotImplementedError('logAccuracy: Selected base ({0}) for logarithms not supported.'.format(base) + + 'Supported values are {0}'.format(supportedbases)) + logacc = logfuncs[base](pred/obse) + if mask: + return logfuncs[base](pred/obse) + else: + + return logfuncs[base](pred/obse) -def medSymAccuracy(predicted, observed, mfunc=np.median, method='log'): +def medSymAccuracy(predicted, observed, mfunc=np.median, method=None): '''Median Symmetric Accuracy: Scaled measure of accuracy that is not biased to over- or under-predictions. The accuracy ratio is given by (prediction/observation), to avoid the bias inherent in mean/median percentage error @@ -426,10 +640,39 @@ def medSymAccuracy(predicted, observed, mfunc=np.median, method='log'): Reference: Morley, S.K. (2016), Alternatives to accuracy and bias metrics based on percentage errors for radiation belt modeling applications, Los Alamos National Laboratory Report, LA-UR-15-24592. + + Parameters + ========== + predicted : array-like + Array-like (list, numpy array, etc.) of predictions + observed : array-like + Array-like (list, numpy array, etc.) of observed values of scalar quantity + + Other Parameters + ================ + method : string, optional + Method to use for calculating the median symmetric accuracy (MSA). Options are 'log' which uses the median of + the re-exponentiated absolute log accuracy, 'UPE' which calculates MSA using the unsigned percentage error, and + None (default), in which case the method is implemented as described above. The UPE method has reduced accuracy + compared to the other methods and is included primarily for testing purposes. + + Returns + ======= + msa : array + Array of median symmetric accuracy ''' pred = _maskSeries(predicted) obse = _maskSeries(observed) - if method=='UPE': + if method is None: + absLogAcc = np.abs(logAccuracy(pred, obse, base=2)) #is this different from the method above for large series?? + symAcc = np.exp2(mfunc(absLogAcc)) + msa = 100*(symAcc-1) + elif method=='log': + ##median(log(Q)) method + absLogAcc = np.abs(logAccuracy(pred, obse, base=2)) + symAcc = mfunc(np.exp2(absLogAcc)) + msa = 100*(symAcc-1) + elif method=='UPE': ##unsigned percentage error method PleO = pred >= obse OltP = np.logical_not(PleO) @@ -438,17 +681,9 @@ def medSymAccuracy(predicted, observed, mfunc=np.median, method='log'): unsRelErr[OltP] = (obse[OltP]-pred[OltP])/pred[OltP] unsPercErr = unsRelErr*100 msa = mfunc(unsPercErr.compressed()) - elif method=='log': - ##median(log(Q)) method - absLogAcc = np.abs(logAccuracy(pred, obse, base=2)) - symAcc = mfunc(np.exp2(absLogAcc)) - msa = 100*(symAcc-1) else: - absLogAcc = np.abs(logAccuracy(pred, obse, base=2)) #is this different from the method above for large series?? - symAcc = np.exp2(mfunc(absLogAcc)) - msa = 100*(symAcc-1) + raise NotImplementedError('medSymAccuracy: invalid method {0}. Valid options are None, "log" or "UPE".'.format(method)) - #raise ValueError('method kwarg should take the value "log" or "UPE".') return msa From 09ccc4d1fe0670a54f8d654adf0830e3a074353e Mon Sep 17 00:00:00 2001 From: Steve Morley Date: Wed, 26 Sep 2018 18:30:24 -0600 Subject: [PATCH 2/4] test_verify.py: expanded test suite coverage Coverage now up to 70%. Added line coverage, but also new tests for bad inputs, regressions, and expected values. --- test_verify.py | 119 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) diff --git a/test_verify.py b/test_verify.py index 2909901..affc7d7 100644 --- a/test_verify.py +++ b/test_verify.py @@ -3,6 +3,7 @@ import unittest import copy +import warnings import numpy as np import numpy.testing as npt import verify @@ -38,6 +39,18 @@ def test_direct_array(self): ctab = verify.Contingency2x2(np.asarray(self.res)) npt.assert_array_equal(self.res, ctab) + def test_direct_array_floats(self): + '''check construction from direct provision of rates as array, force floating point''' + ctab = verify.Contingency2x2(np.asarray(self.res), dtype=np.float) + npt.assert_array_equal(np.asarray(self.res, dtype=np.float), ctab) + + def test_direct_array3D_raises(self): + '''check construction raises when given rates from non-2D array''' + self.assertRaises(ValueError, verify.Contingency2x2, np.ones([2,2,2])) + + def test_direct_array5x2_raises(self): + '''check construction raises when given rates from wrong shape array''' + self.assertRaises(ValueError, verify.Contingency2x2, [self.predlist,self.obslist]) class contingency2x2_calc(unittest.TestCase): def setUp(self): @@ -108,6 +121,14 @@ def test_Finley_PSS(self): ctab = verify.Contingency2x2(self.Finley) self.assertAlmostEqual(ctab.peirce(), 0.523, places=3) + def test_Finley_PSS_CIWald_regress(self): + ctab = verify.Contingency2x2(self.Finley) + npt.assert_array_almost_equal(ctab.peirce(ci='Wald'), [0.5228568171454651, 0.13669651496274715], decimal=5) + + def test_Finley_PSS_CIAgresti_regress(self): + ctab = verify.Contingency2x2(self.Finley) + npt.assert_array_almost_equal(ctab.peirce(ci='AC'), [0.5228568171454651, 0.13187940039376933], decimal=5) + def test_Finley_GSS(self): ctab = verify.Contingency2x2(self.Finley) self.assertAlmostEqual(ctab.equitableThreat(), 0.216, places=3) @@ -123,6 +144,11 @@ def test_Finley_MC(self): self.assertEqual(ctab_const_no.PC(), ctab.majorityClassFraction()) self.assertAlmostEqual(ctab.majorityClassFraction(), 0.982, places=3) + def test_BootstrapCI_regress(self): + '''Test bootstrap CI calculation gives expected answer for given input''' + ctab = verify.Contingency2x2.fromBoolean(self.predlist*30, self.obslist*30) + gotval, gotci = ctab.PC(ci='bootstrap') + npt.assert_array_almost_equal([gotval, gotci[0], gotci[1]], [0.4, 0.32, 0.48]) class contingencyNxN_tests(unittest.TestCase): def test_NxNto2x2(self): @@ -134,6 +160,7 @@ def test_NxNto2x2(self): class perfectContinuous(unittest.TestCase): + '''Tests of continuous metrics using perfects forecasts''' def setUp(self): super(self.__class__, self).setUp() self.predvec = [8, 9, 11, 10, 10, 11, 15, 19, 17, 16, 12, 8, 6, 7, 9, 15, 16, 17, 24, 28, 32, 28, 20] @@ -165,8 +192,22 @@ def test_percBetter(self): badmodel = verify._maskSeries(self.predvec) - 1 self.assertEqual(verify.percBetter(self.predvec, badmodel, self.obsvec), 100) + def test_logAccuracy(self): + '''Log accuracy of perfect prediction is zero''' + ans = np.zeros_like(self.obsvec) + npt.assert_array_equal(verify.logAccuracy(self.predvec, self.obsvec, mask=False), ans) + + def test_APE(self): + '''Absolute percentage error of perfect prediction is zero''' + ans = np.zeros_like(self.obsvec) + npt.assert_array_equal(verify.absPercError(self.predvec, self.obsvec), ans) + + def test_symSignedBias(self): + '''Symmetric signed percentage bias of perfect prediction is zero''' + self.assertEqual(verify.symmetricSignedBias(self.predvec, self.obsvec), 0) class relativeContinuous(unittest.TestCase): + '''Tests of relative error metrics for known relative error''' def setUp(self): super(self.__class__, self).setUp() self.predvec = [170]*10 @@ -175,6 +216,16 @@ def setUp(self): def tearDown(self): super(self.__class__, self).tearDown() + def test_APE1(self): + '''Test estimation of absolute percentage error (positive)''' + ans = np.ones_like(self.obsvec) + npt.assert_array_equal(verify.absPercError(self.predvec, self.obsvec), ans*70) + + def test_APE2(self): + '''Test estimation of absolute percentage error (negative)''' + ans = np.ones_like(self.obsvec) + npt.assert_array_equal(verify.absPercError(np.asarray(self.predvec)*-1, np.asarray(self.obsvec)*-1), ans*70) + def test_meanPercentageError(self): '''Test estimation of mean percentage error''' self.assertEqual(verify.meanPercentageError(self.predvec, self.obsvec), 70) @@ -197,6 +248,7 @@ def test_medSymAccuracy3(self): class otherContinuous(unittest.TestCase): + '''Other tests of continuous error metrics''' def setUp(self): super(self.__class__, self).setUp() self.predalt = [-2, 2, -2, 2, -2, 2, -2, 2] @@ -253,8 +305,58 @@ def test_medAbsError_equal_meanAbsError_const(self): obsvec = [val-diff]*n_elements npt.assert_array_equal(verify.medAbsError(predvec, obsvec), verify.meanAbsError(predvec, obsvec)) + def test_medSymAcc_AllMethodsEqual1(self): + '''median symmetric accuracy is within precision for different methods''' + np.random.seed(24601) + predvec = np.arange(1,40) + obsvec = predvec+np.random.random(len(predvec)) + npt.assert_array_almost_equal(verify.medSymAccuracy(predvec, obsvec), verify.medSymAccuracy(predvec, obsvec, method='log')) + + def test_medSymAcc_AllMethodsEqual2(self): + '''median symmetric accuracy is within precision for different methods''' + np.random.seed(24601) + predvec = np.arange(1,40) + obsvec = predvec+np.random.random(len(predvec)) + npt.assert_array_almost_equal(verify.medSymAccuracy(predvec, obsvec), verify.medSymAccuracy(predvec, obsvec, method='UPE')) + +class badInputs(unittest.TestCase): + '''Tests of that should raise errors (or related cases)''' + def setUp(self): + super(self.__class__, self).setUp() + self.predvec = np.arange(1,40) + self.obsvec = self.predvec+1 + self.Finley = [[28,72],[23,2680]] + + def tearDown(self): + super(self.__class__, self).tearDown() + + def test_medSymAcc_raises(self): + '''median symmetric accuracy raises an error when an invalid method is selected''' + self.assertRaises(NotImplementedError, verify.medSymAccuracy, self.predvec, self.obsvec, method='NotAMethod') + + def test_logAcc_base_raises(self): + '''log accuracy raises an error when an invalid base is selected''' + self.assertRaises(NotImplementedError, verify.logAccuracy, self.predvec, self.obsvec, base=7.4) + self.assertRaises(NotImplementedError, verify.logAccuracy, self.predvec, self.obsvec, base='SevenPoint4') + self.assertRaises(NotImplementedError, verify.logAccuracy, self.predvec, self.obsvec, base=np.e) + + def test_logAcc_negative_raises(self): + '''log accuracy raises an error when negative values are given''' + self.assertRaises(ValueError, verify.logAccuracy, self.predvec*-1, self.obsvec, mask=False) + + def test_logAcc_negative_noRaiseWhenMasked(self): + '''log accuracy masks all values when predicted array is all negative''' + with warnings.catch_warnings(): #squelch RuntimeWarnings as this will raise w/ msg "invalid value encountered in log10" + warnings.simplefilter("ignore") + self.assertTrue(verify.logAccuracy(self.predvec*-1, self.obsvec, mask=True).mask.all()) + + def test_BootstrapCI_failsWithoutBooleanCreate(self): + '''Test bootstrap CI calculation raises an error when called on a table without input booleans''' + ctab = verify.Contingency2x2(self.Finley) + self.assertRaises(AttributeError, ctab.POD, ci='bootstrap') class individualContinuous(unittest.TestCase): + '''Tests for continuous metrics using single element prediction/observation vectors''' def test_forecastError_spanzero_list(self): pred, obs = [-1], [1] self.assertEqual(verify.forecastError(pred, obs, full=False), -2) @@ -269,6 +371,23 @@ def test_forecastError_full(self): npt.assert_array_equal(newpred, pred) npt.assert_array_equal(newobs, obs) +class precisionMeasures(unittest.TestCase): + '''Tests for measures of spread''' + def test_Sn_expected_Gaussian_even(self): + '''Test that Sn estimator has a mean estimated value consistent with expectation''' + np.random.seed(24601) + n_elements = 1500 + n_repeats = 5 + sns = [verify.Sn(np.random.randn(n_elements)) for i in range(n_repeats)] + npt.assert_almost_equal(np.mean(sns), 1, decimal=2) + + def test_Sn_expected_Gaussian_odd(self): + '''Test that Sn estimator has a mean estimated value consistent with expectation''' + np.random.seed(24601) + n_elements = 1499 + n_repeats = 5 + sns = [verify.Sn(np.random.randn(n_elements)) for i in range(n_repeats)] + npt.assert_almost_equal(np.mean(sns), 1, decimal=2) if __name__ == '__main__': unittest.main() From 44b178dc74ae9bfe57a2bc83a58abd1afef2ed3a Mon Sep 17 00:00:00 2001 From: Steve Morley Date: Wed, 26 Sep 2018 18:47:58 -0600 Subject: [PATCH 3/4] docs: Initial commit of docs folder for Sphinx documentation Docs folder was missed from earlier commits, but is required to work on the gh-pages branch. --- docs/Makefile | 216 ++++++++++++++++++++++++++++++++++++ docs/conf.py | 287 ++++++++++++++++++++++++++++++++++++++++++++++++ docs/index.rst | 23 ++++ docs/make.bat | 263 ++++++++++++++++++++++++++++++++++++++++++++ docs/verify.rst | 52 +++++++++ 5 files changed, 841 insertions(+) create mode 100644 docs/Makefile create mode 100644 docs/conf.py create mode 100644 docs/index.rst create mode 100644 docs/make.bat create mode 100644 docs/verify.rst diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..6c9dcb8 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,216 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = ../../PyForecastTools-docs + +# User-friendly check for sphinx-build +ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) +$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) +endif + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " applehelp to make an Apple Help Book" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " xml to make Docutils-native XML files" + @echo " pseudoxml to make pseudoxml-XML files for display purposes" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + @echo " coverage to run coverage check of the documentation (if enabled)" + +.PHONY: clean +clean: + rm -rf $(BUILDDIR)/* + +.PHONY: html +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +.PHONY: dirhtml +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +.PHONY: singlehtml +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +.PHONY: pickle +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +.PHONY: json +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +.PHONY: htmlhelp +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +.PHONY: qthelp +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/PyForecastTools.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/PyForecastTools.qhc" + +.PHONY: applehelp +applehelp: + $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp + @echo + @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." + @echo "N.B. You won't be able to view it unless you put it in" \ + "~/Library/Documentation/Help or install it in your application" \ + "bundle." + +.PHONY: devhelp +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/PyForecastTools" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/PyForecastTools" + @echo "# devhelp" + +.PHONY: epub +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +.PHONY: latex +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +.PHONY: latexpdf +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +.PHONY: latexpdfja +latexpdfja: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through platex and dvipdfmx..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +.PHONY: text +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +.PHONY: man +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +.PHONY: texinfo +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +.PHONY: info +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +.PHONY: gettext +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +.PHONY: changes +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +.PHONY: linkcheck +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +.PHONY: doctest +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." + +.PHONY: coverage +coverage: + $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage + @echo "Testing of coverage in the sources finished, look at the " \ + "results in $(BUILDDIR)/coverage/python.txt." + +.PHONY: xml +xml: + $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml + @echo + @echo "Build finished. The XML files are in $(BUILDDIR)/xml." + +.PHONY: pseudoxml +pseudoxml: + $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml + @echo + @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..55daec6 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,287 @@ +# -*- coding: utf-8 -*- +# +# PyForecastTools documentation build configuration file, created by +# sphinx-quickstart on Mon Jun 11 16:05:02 2018. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys +import os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +#sys.path.insert(0, os.path.abspath('.')) + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +#needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.doctest', + 'sphinx.ext.autosummary', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'PyForecastTools' +copyright = u'2018, Los Alamos National Security, LLC' +author = u'Steven K. Morley' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = u'1.0' +# The full version, including alpha/beta/rc tags. +release = u'1.0' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = ['_build'] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +#keep_warnings = False + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'alabaster' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (relative to this directory) to use as a favicon of +# the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +#html_extra_path = [] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_domain_indices = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +#html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +#html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = None + +# Language to be used for generating the HTML full-text search index. +# Sphinx supports the following languages: +# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' +# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' +#html_search_language = 'en' + +# A dictionary with options for the search language support, empty by default. +# Now only 'ja' uses this config value +#html_search_options = {'type': 'default'} + +# The name of a javascript file (relative to the configuration directory) that +# implements a search results scorer. If empty, the default will be used. +#html_search_scorer = 'scorer.js' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'PyForecastToolsdoc' + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { +# The paper size ('letterpaper' or 'a4paper'). +#'papersize': 'letterpaper', + +# The font size ('10pt', '11pt' or '12pt'). +#'pointsize': '10pt', + +# Additional stuff for the LaTeX preamble. +#'preamble': '', + +# Latex figure (float) alignment +#'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'PyForecastTools.tex', u'PyForecastTools Documentation', + u'Steven K. Morley', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# If true, show page references after internal links. +#latex_show_pagerefs = False + +# If true, show URL addresses after external links. +#latex_show_urls = False + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'pyforecasttools', u'PyForecastTools Documentation', + [author], 1) +] + +# If true, show URL addresses after external links. +#man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'PyForecastTools', u'PyForecastTools Documentation', + author, 'PyForecastTools', 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +#texinfo_appendices = [] + +# If false, no module index is generated. +#texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +#texinfo_no_detailmenu = False diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..655442a --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,23 @@ +.. PyForecastTools documentation master file, created by + sphinx-quickstart on Mon Jun 11 16:05:02 2018. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +PyForecastTools documentation +============================= + +Contents: + +.. toctree:: + :maxdepth: 2 + + verify + + +Indices and tables +------------------ + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..e3b2712 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,263 @@ +@ECHO OFF + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set BUILDDIR=_build +set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . +set I18NSPHINXOPTS=%SPHINXOPTS% . +if NOT "%PAPER%" == "" ( + set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% + set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% +) + +if "%1" == "" goto help + +if "%1" == "help" ( + :help + echo.Please use `make ^` where ^ is one of + echo. html to make standalone HTML files + echo. dirhtml to make HTML files named index.html in directories + echo. singlehtml to make a single large HTML file + echo. pickle to make pickle files + echo. json to make JSON files + echo. htmlhelp to make HTML files and a HTML help project + echo. qthelp to make HTML files and a qthelp project + echo. devhelp to make HTML files and a Devhelp project + echo. epub to make an epub + echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter + echo. text to make text files + echo. man to make manual pages + echo. texinfo to make Texinfo files + echo. gettext to make PO message catalogs + echo. changes to make an overview over all changed/added/deprecated items + echo. xml to make Docutils-native XML files + echo. pseudoxml to make pseudoxml-XML files for display purposes + echo. linkcheck to check all external links for integrity + echo. doctest to run all doctests embedded in the documentation if enabled + echo. coverage to run coverage check of the documentation if enabled + goto end +) + +if "%1" == "clean" ( + for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i + del /q /s %BUILDDIR%\* + goto end +) + + +REM Check if sphinx-build is available and fallback to Python version if any +%SPHINXBUILD% 1>NUL 2>NUL +if errorlevel 9009 goto sphinx_python +goto sphinx_ok + +:sphinx_python + +set SPHINXBUILD=python -m sphinx.__init__ +%SPHINXBUILD% 2> nul +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +:sphinx_ok + + +if "%1" == "html" ( + %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/html. + goto end +) + +if "%1" == "dirhtml" ( + %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. + goto end +) + +if "%1" == "singlehtml" ( + %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. + goto end +) + +if "%1" == "pickle" ( + %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the pickle files. + goto end +) + +if "%1" == "json" ( + %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the JSON files. + goto end +) + +if "%1" == "htmlhelp" ( + %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run HTML Help Workshop with the ^ +.hhp project file in %BUILDDIR%/htmlhelp. + goto end +) + +if "%1" == "qthelp" ( + %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run "qcollectiongenerator" with the ^ +.qhcp project file in %BUILDDIR%/qthelp, like this: + echo.^> qcollectiongenerator %BUILDDIR%\qthelp\PyForecastTools.qhcp + echo.To view the help file: + echo.^> assistant -collectionFile %BUILDDIR%\qthelp\PyForecastTools.ghc + goto end +) + +if "%1" == "devhelp" ( + %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. + goto end +) + +if "%1" == "epub" ( + %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The epub file is in %BUILDDIR%/epub. + goto end +) + +if "%1" == "latex" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "latexpdf" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + cd %BUILDDIR%/latex + make all-pdf + cd %~dp0 + echo. + echo.Build finished; the PDF files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "latexpdfja" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + cd %BUILDDIR%/latex + make all-pdf-ja + cd %~dp0 + echo. + echo.Build finished; the PDF files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "text" ( + %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The text files are in %BUILDDIR%/text. + goto end +) + +if "%1" == "man" ( + %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The manual pages are in %BUILDDIR%/man. + goto end +) + +if "%1" == "texinfo" ( + %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. + goto end +) + +if "%1" == "gettext" ( + %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The message catalogs are in %BUILDDIR%/locale. + goto end +) + +if "%1" == "changes" ( + %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes + if errorlevel 1 exit /b 1 + echo. + echo.The overview file is in %BUILDDIR%/changes. + goto end +) + +if "%1" == "linkcheck" ( + %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck + if errorlevel 1 exit /b 1 + echo. + echo.Link check complete; look for any errors in the above output ^ +or in %BUILDDIR%/linkcheck/output.txt. + goto end +) + +if "%1" == "doctest" ( + %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest + if errorlevel 1 exit /b 1 + echo. + echo.Testing of doctests in the sources finished, look at the ^ +results in %BUILDDIR%/doctest/output.txt. + goto end +) + +if "%1" == "coverage" ( + %SPHINXBUILD% -b coverage %ALLSPHINXOPTS% %BUILDDIR%/coverage + if errorlevel 1 exit /b 1 + echo. + echo.Testing of coverage in the sources finished, look at the ^ +results in %BUILDDIR%/coverage/python.txt. + goto end +) + +if "%1" == "xml" ( + %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The XML files are in %BUILDDIR%/xml. + goto end +) + +if "%1" == "pseudoxml" ( + %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. + goto end +) + +:end diff --git a/docs/verify.rst b/docs/verify.rst new file mode 100644 index 0000000..02eb5e1 --- /dev/null +++ b/docs/verify.rst @@ -0,0 +1,52 @@ +############################################################ +PyForecastTools - Model Validation and Forecast Verification +############################################################ + +Verify - Core metrics and classes +================================= + +.. currentmodule:: verify + +- `Metrics`_ +- `Contingency Tables`_ + +Metrics +------- +.. rubric:: Functions + +.. autosummary:: + :toctree: autosummary + + skill + percBetter + bias + meanPercentageError + medianLogAccuracy + symmetricSignedBias + accuracy + medSymAccuracy + meanSquaredError + RMSE + meanAPE + meanAbsError + medAbsError + scaledAccuracy + nRMSE + + +Contingency Tables +------------------ +.. rubric:: Classes + +.. autosummary:: + :template: clean_class.rst + :toctree: autosummary + + ContingencyNxN + Contingency2x2 + +Function/Class Documentation +---------------------------- +.. automodule:: verify + :members: + From 1ec0900ee504b59c637b578a2f0e012600451389 Mon Sep 17 00:00:00 2001 From: Brian Larsen Date: Tue, 2 Oct 2018 09:58:01 -0600 Subject: [PATCH 4/4] fixed typo in scaledAccuracy. predcted->predicted --- verify/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/verify/__init__.py b/verify/__init__.py index 81dba6d..af9ee9a 100644 --- a/verify/__init__.py +++ b/verify/__init__.py @@ -373,7 +373,7 @@ def scaledAccuracy(predicted, observed): 'MdSymAcc': medSymAccuracy} out = dict() for met in metrics: - out[met] = metrics[met](predcted, observed) + out[met] = metrics[met](predicted, observed) return out