From 139f474118b952fd3cb9843dcc8bdd8dfaa95581 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Sun, 13 Apr 2014 19:31:29 -0700 Subject: [PATCH 01/13] fix bug in read_forcing.py that failed to read last yearlty forcing file. Fixes issue #6 --- rvic/read_forcing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rvic/read_forcing.py b/rvic/read_forcing.py index 14bdf2a..8cc99c2 100644 --- a/rvic/read_forcing.py +++ b/rvic/read_forcing.py @@ -56,7 +56,7 @@ def __init__(self, path, file_str, time_fld, liq_flds, # yearly files elif len(start) == 1: - for year in xrange(start[0], end[0]): + for year in xrange(start[0], end[0]+1): self.files.append(os.path.join(self.path, file_str.replace('$YYYY', "{0:04d}".format(year)))) From d4a4f44e516c63314a3332149b87247e7c40a716 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Mon, 14 Apr 2014 22:39:03 -0700 Subject: [PATCH 02/13] enable variable expansion in configuration files --- rvic/core/config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rvic/core/config.py b/rvic/core/config.py index 5ec4a30..c58d1fb 100644 --- a/rvic/core/config.py +++ b/rvic/core/config.py @@ -3,6 +3,7 @@ """ +import os from ConfigParser import SafeConfigParser @@ -64,7 +65,7 @@ def config_type(value): try: return float(value) except: - return value + return os.path.expandvars(value) else: try: return map(float, val_list) From e5e8c7bcc334e0e215a0a6b33f548444865424be Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Mon, 14 Apr 2014 23:50:08 -0700 Subject: [PATCH 03/13] add run_tests.py and reorg test dir --- tests/run_tests.py | 92 ++++++++++++++++++++++++++++++ tests/rvic_system_test.py | 11 ---- tests/{ => unit}/rvic_unit_test.py | 0 3 files changed, 92 insertions(+), 11 deletions(-) create mode 100755 tests/run_tests.py delete mode 100644 tests/rvic_system_test.py rename tests/{ => unit}/rvic_unit_test.py (100%) diff --git a/tests/run_tests.py b/tests/run_tests.py new file mode 100755 index 0000000..c58ba62 --- /dev/null +++ b/tests/run_tests.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python +"""RVIC command line testing interface""" + +from __future__ import print_function +import argparse +import pytest +import cProfile +import pstats +import StringIO +from rvic import convert, convolution, parameters +from rvic.core.config import read_config + + +# -------------------------------------------------------------------- # +def main(): + """ + Run RVIC tests + """ + # Parse arguments + parser = argparse.ArgumentParser(description='Test script for RVIC') + + parser.add_argument("test_set", type=str, + help="Test set to run", + choices=['all', 'unit', 'examples'], + default=['all'], nargs='+') + parser.add_argument("--examples", type=str, + help="examples configuration file", + default='examples/examples.cfg') + args = parser.parse_args() + + print('Running Test Set: {0}'.format(args.test_set)) + + if any(i in ['all', 'unit'] for i in args.test_set): + # run unit tests + pytest.main('-x unit') + if any(i in ['all', 'examples'] for i in args.test_set): + run_examples(args.examples) + return +# -------------------------------------------------------------------- # + + +# -------------------------------------------------------------------- # +def run_examples(config_file): + """ Run examples from config file """ + # ---------------------------------------------------------------- # + # Read Configuration files + config_dict = read_config(config_file) + # ---------------------------------------------------------------- # + + # ---------------------------------------------------------------- # + # run tests + num_tests = len(config_dict.keys()) + + for i, (test, test_dict) in enumerate(config_dict.iteritems()): + print("".center(100, '-')) + print("Starting Test #{0} of {1}: {2}".format(i+1, num_tests, + test).center(100)) + print("".center(100, '-')) + pr = cProfile.Profile() + pr.enable() + + if test_dict['function'] == 'convert': + convert.convert(test_dict['config_file']) + elif test_dict['function'] == 'convolution': + convolution.convolution(test_dict['config_file']) + elif test_dict['function'] == 'parameters': + parameters.parameters(test_dict['config_file']) + else: + raise ValueError('Unknow function variable: ' + '{0}'.format(test_dict['function'])) + + pr.disable() + s = StringIO.StringIO() + sortby = 'cumulative' + ps = pstats.Stats(pr, stream=s).sort_stats(sortby) + ps.print_stats() + + print("".center(100, '-')) + print("Done With Test #{0} of {1}: {2}".format(i+1, num_tests, + test).center(100)) + print(".....Printing Profile Information.....".center(100)) + print("".center(100, '-')) + print(s.getvalue()) + print("".center(100, '-')) + + return + + +# -------------------------------------------------------------------- # +if __name__ == "__main__": + main() +# -------------------------------------------------------------------- # diff --git a/tests/rvic_system_test.py b/tests/rvic_system_test.py deleted file mode 100644 index 7ae9c50..0000000 --- a/tests/rvic_system_test.py +++ /dev/null @@ -1,11 +0,0 @@ -#!/opt/local/bin/python -""" -rvic_system_test.py - -Set to run with pytest, - -Usage: py.test (from RVIC or test directory) - -Concept: Run a handfull of possible configurations of the RVIC modules. -""" - diff --git a/tests/rvic_unit_test.py b/tests/unit/rvic_unit_test.py similarity index 100% rename from tests/rvic_unit_test.py rename to tests/unit/rvic_unit_test.py From 6f23a4dd922521ca599c065f643d0ec9a972220a Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Mon, 14 Apr 2014 23:53:16 -0700 Subject: [PATCH 04/13] update travis.yml to include unit tests only --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 24abfda..660863e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,4 +26,6 @@ script: - pip freeze - echo $PATH - python -c 'import rvic; print rvic.__file__' - - py.test # erroring here for unknown path reasons... + - cd tests + - pwd + - python run_tests.py unit From 136a9f15f9b529bcba872b6716867bd24a0720b5 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Tue, 15 Apr 2014 00:06:23 -0700 Subject: [PATCH 05/13] add pytest to requirements --- ci/requirements-2.7.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/requirements-2.7.txt b/ci/requirements-2.7.txt index 1214390..7aabeab 100644 --- a/ci/requirements-2.7.txt +++ b/ci/requirements-2.7.txt @@ -2,3 +2,4 @@ matplotlib==1.3.1 netCDF4==1.0.8 numpy==1.8.1 scipy==0.13.3 +pytest==2.5.2 From deea9ade931f4000b8ada30a3232c254899afb74 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Tue, 15 Apr 2014 00:11:35 -0700 Subject: [PATCH 06/13] add pandas requirement --- ci/requirements-2.7-dev.txt | 1 + ci/requirements-2.7.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/ci/requirements-2.7-dev.txt b/ci/requirements-2.7-dev.txt index 5148a16..c1220c8 100644 --- a/ci/requirements-2.7-dev.txt +++ b/ci/requirements-2.7-dev.txt @@ -3,4 +3,5 @@ matplotlib==1.3.1 netCDF4==1.0.8 numpy==1.8.1 scipy==0.13.3 +pandas==0.13.1 pytest==2.5.2 diff --git a/ci/requirements-2.7.txt b/ci/requirements-2.7.txt index 7aabeab..cfe6cda 100644 --- a/ci/requirements-2.7.txt +++ b/ci/requirements-2.7.txt @@ -2,4 +2,5 @@ matplotlib==1.3.1 netCDF4==1.0.8 numpy==1.8.1 scipy==0.13.3 +pandas==0.13.1 pytest==2.5.2 From e0bf3396d951f52f07ff3740e769aa5fb18d44ba Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Wed, 16 Apr 2014 22:03:47 -0700 Subject: [PATCH 07/13] Minor fixes to issues found during testing. - Close logger and resume printing to stdout and stderr - Improved parsing of config file (specifically individual float and int values) - Fix to invalid char function during reading of station names. Now only allow alpha numeric characters and a selection of underscore and period type chars. --- rvic/convert.py | 4 +++- rvic/convolution.py | 4 +++- rvic/core/config.py | 38 ++++++++++++++++++++++++++++++++++---- rvic/core/log.py | 12 ++++++++++++ rvic/core/share.py | 3 +++ rvic/parameters.py | 9 ++++++--- 6 files changed, 61 insertions(+), 9 deletions(-) diff --git a/rvic/convert.py b/rvic/convert.py index c8afb63..0007619 100644 --- a/rvic/convert.py +++ b/rvic/convert.py @@ -3,7 +3,7 @@ """ from logging import getLogger -from core.log import init_logger, LOG_NAME +from core.log import init_logger, close_logger, LOG_NAME from core.utilities import make_directories, copy_inputs, read_domain from core.utilities import tar_inputs from core.convert import read_station_file, read_uhs_files, move_domain @@ -143,6 +143,8 @@ def uhs2param_final(outlets, dom_data, new_dom_data, config_dict, directories): log.info('Location of Inputs: %s', inputs_tar) log.info('Location of Log: %s', log_tar) log.info('Location of Parmeter File %s', param_file) + + close_logger() # ---------------------------------------------------------------- # return # -------------------------------------------------------------------- # diff --git a/rvic/convolution.py b/rvic/convolution.py index e3a2590..173dade 100644 --- a/rvic/convolution.py +++ b/rvic/convolution.py @@ -15,7 +15,7 @@ """ import os from logging import getLogger -from core.log import init_logger, LOG_NAME +from core.log import init_logger, close_logger, LOG_NAME from core.utilities import make_directories, read_domain from core.utilities import write_rpointer, tar_inputs from core.variables import Rvar @@ -347,6 +347,8 @@ def convolution_final(time_handle, hist_tapes): log.info('Done with rvic convolution.') log.info('Location of Log: %s', log_tar) + + close_logger() # ---------------------------------------------------------------- # return # -------------------------------------------------------------------- # diff --git a/rvic/core/config.py b/rvic/core/config.py index c58d1fb..191d645 100644 --- a/rvic/core/config.py +++ b/rvic/core/config.py @@ -61,14 +61,44 @@ def config_type(value): return False elif value in ['none', 'None', 'NONE', '']: return None + elif isint(value): + return int(value) + elif isfloat(value): + return float(value) else: - try: - return float(value) - except: - return os.path.expandvars(value) + return os.path.expandvars(value) else: try: return map(float, val_list) + except: + pass + try: + return map(int, val_list) except: return val_list # -------------------------------------------------------------------- # + + +# -------------------------------------------------------------------- # +def isfloat(x): + """Test of value is a float""" + try: + a = float(x) + except ValueError: + return False + else: + return True +# -------------------------------------------------------------------- # + + +# -------------------------------------------------------------------- # +def isint(x): + """Test if value is an integer""" + try: + a = float(x) + b = int(a) + except ValueError: + return False + else: + return a == b +# -------------------------------------------------------------------- # diff --git a/rvic/core/log.py b/rvic/core/log.py index 07d491b..e612f42 100644 --- a/rvic/core/log.py +++ b/rvic/core/log.py @@ -78,3 +78,15 @@ def init_logger(log_dir='./', log_level='DEBUG', verbose=False): return logger # -------------------------------------------------------------------- # + + +def close_logger(): + """Close the handlers of the logger""" + log = logging.getLogger(LOG_NAME) + x = list(log.handlers) + for i in x: + log.removeHandler(i) + i.flush() + i.close() + sys.stdout = sys.__stdout__ + sys.stderr = sys.__stderr__ diff --git a/rvic/core/share.py b/rvic/core/share.py index 410db55..b735789 100644 --- a/rvic/core/share.py +++ b/rvic/core/share.py @@ -3,6 +3,7 @@ """ import sys import socket +import string import time as time_mod from collections import OrderedDict from netCDF4 import default_fillvals @@ -64,6 +65,8 @@ 5: ['360_day'], 6: ['julian']} +VALID_CHARS = "-_. %s%s" % (string.ascii_letters, string.digits) + # ----------------------- NETCDF VARIABLES --------------------------------- # class NcGlobals: diff --git a/rvic/parameters.py b/rvic/parameters.py index 922c37a..36694f1 100644 --- a/rvic/parameters.py +++ b/rvic/parameters.py @@ -6,9 +6,9 @@ import pandas as pd from collections import OrderedDict from logging import getLogger -from core.log import init_logger, LOG_NAME +from core.log import init_logger, close_logger, LOG_NAME from core.mpi import LoggingPool -from core.utilities import make_directories, copy_inputs, strip_non_ascii +from core.utilities import make_directories, copy_inputs, strip_invalid_char from core.utilities import read_netcdf, tar_inputs, latlon2yx from core.utilities import check_ncvars, clean_file, read_domain from core.aggregate import make_agg_pairs, aggregate @@ -123,7 +123,8 @@ def gen_uh_init(config_file): if 'names' in pour_points: pour_points.fillna(inplace=True, value='unknown') for i, name in enumerate(pour_points.names): - pour_points.names[i] = strip_non_ascii(name) + pour_points.names[i] = strip_invalid_char(name) + pour_points.drop_duplicates(inplace=True) pour_points.dropna() except Exception as e: @@ -471,6 +472,8 @@ def gen_uh_final(outlets, dom_data, config_dict, directories): log.info('Location of Inputs: %s', inputs_tar) log.info('Location of Log: %s', log_tar) log.info('Location of Parmeter File %s', param_file) + + close_logger() # ---------------------------------------------------------------- # return # -------------------------------------------------------------------- # From f1fabd8ca64fd31451f378bd2d59481976545a39 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Wed, 16 Apr 2014 22:06:56 -0700 Subject: [PATCH 08/13] Fixes to issues found during testing. - Add missing variables when not subsetting unit hydrographs - Add function for handling disallowed characters (see previous commit) - Remove compression args for character arrays (outlet_names) - Replace source area variable in sample configuration file. --- config/rvic_parameters_example.cfg | 1 + rvic/core/aggregate.py | 3 --- rvic/core/param_file.py | 4 ++++ rvic/core/utilities.py | 13 ++++++++++--- rvic/core/write.py | 2 +- 5 files changed, 16 insertions(+), 7 deletions(-) diff --git a/config/rvic_parameters_example.cfg b/config/rvic_parameters_example.cfg index 581c6fb..01f514f 100644 --- a/config/rvic_parameters_example.cfg +++ b/config/rvic_parameters_example.cfg @@ -98,6 +98,7 @@ LATITUDE_VAR: lat FLOW_DISTANCE_VAR: Flow_Distance FLOW_DIRECTION_VAR: Flow_Direction BASIN_ID_VAR: Basin_ID +SOURCE_AREA_VAR: Source_Area #-- Velocity and diffusion --# # The velocity and diffusion parameters may either be specified as variables in diff --git a/rvic/core/aggregate.py b/rvic/core/aggregate.py index c981bdb..424553b 100644 --- a/rvic/core/aggregate.py +++ b/rvic/core/aggregate.py @@ -4,7 +4,6 @@ """ import numpy as np -from scipy.spatial import cKDTree from collections import OrderedDict from share import FILLVALUE_F from utilities import find_nearest, latlon2yx @@ -21,8 +20,6 @@ # -------------------------------------------------------------------- # # Find target cells for pour points def make_agg_pairs(pour_points, dom_data, fdr_data, config_dict): -# def make_agg_pairs(lons, lats, dom_lon, dom_lat, dom_ids, - # fdr_lons, fdr_lats, fdr_srcarea, agg_type='agg'): """ Group pour points by domain grid outlet cell """ diff --git a/rvic/core/param_file.py b/rvic/core/param_file.py index 7941f99..3ca89c5 100644 --- a/rvic/core/param_file.py +++ b/rvic/core/param_file.py @@ -64,6 +64,10 @@ def finish_params(outlets, dom_data, config_dict, directories): subset_length = routing['BASIN_FLOWDAYS']*SECSPERDAY/routing['OUTPUT_INTERVAL'] log.info('Not subsetting because either SUBSET_DAYS is null or ' 'SUBSET_DAYS Date: Wed, 16 Apr 2014 22:32:56 -0700 Subject: [PATCH 09/13] rename scrip rvic_model to rvic, associated changes to setup.py and README --- README.md | 4 +++- scripts/{rvic_model => rvic} | 0 setup.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) rename scripts/{rvic_model => rvic} (100%) diff --git a/README.md b/README.md index 3adf6f8..0a8a6fc 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # RVIC Streamflow Routing Model -The RVIC Streamflow Routing model is a simple source to sink routing model. The model represents each grid cell by a node in the channel network. Unit hydrographs are developed that described the time distribution of flow from each source grid cell to a corresponding sink grid cell. The development of the unit hydrographs is done as a pre process step (i.e. `rvic_model parameters`). The final step is the convolution of the unit hydrographs with fluxes from a land surface model, typically VIC (i.e. `rvic_model convolution`). +[![Build Status](https://travis-ci.org/jhamman/RVIC.svg?branch=develop)](https://travis-ci.org/jhamman/RVIC) + +The RVIC Streamflow Routing model is a simple source to sink routing model. The model represents each grid cell by a node in the channel network. Unit hydrographs are developed that described the time distribution of flow from each source grid cell to a corresponding sink grid cell. The development of the unit hydrographs is done as a pre process step (i.e. `rvic parameters`). The final step is the convolution of the unit hydrographs with fluxes from a land surface model, typically VIC (i.e. `rvic convolution`). ### Usage See the [RVIC Wiki Page](https://github.com/jhamman/RVIC/wiki/RVIC-Wiki) diff --git a/scripts/rvic_model b/scripts/rvic similarity index 100% rename from scripts/rvic_model rename to scripts/rvic diff --git a/setup.py b/setup.py index 49ea939..1e0fe29 100644 --- a/setup.py +++ b/setup.py @@ -110,7 +110,7 @@ def write_version_py(filename=None): packages=['rvic', 'rvic.core'], platform=['any'], py_modules=['rvic.parameters', 'rvic.convolution', 'rvic.convert'], - scripts=['scripts/rvic_model', 'tools/find_pour_points.py', + scripts=['scripts/rvic', 'tools/find_pour_points.py', 'tools/fraction2domain.bash'], ext_modules=[Extension('rvic_convolution', sources=['rvic/clib/rvic_convolution.c'])]) From 3c5a523df5f1a556d44c930e4943aceda468ff1f Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Wed, 16 Apr 2014 22:36:09 -0700 Subject: [PATCH 10/13] improved run_tests.py, now uses 2 environment variables $RVIC_TEST_DIR and $WORKDIR to specify input and output locations for tests --- tests/run_tests.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/tests/run_tests.py b/tests/run_tests.py index c58ba62..63d3ad2 100755 --- a/tests/run_tests.py +++ b/tests/run_tests.py @@ -2,6 +2,8 @@ """RVIC command line testing interface""" from __future__ import print_function +import os +import textwrap import argparse import pytest import cProfile @@ -10,6 +12,17 @@ from rvic import convert, convolution, parameters from rvic.core.config import read_config +if not os.environ.get('RVIC_TEST_DIR'): + print('\n$RVIC_TEST_DIR not set.') + os.environ["RVIC_TEST_DIR"] = os.path.abspath(os.path.dirname(__file__)) + print('Setting to run_tests.py dir: ' + '{0}\n'.format(os.environ["RVIC_TEST_DIR"])) +if not os.environ.get('WORKDIR'): + print('\n$WORKDIR not set.') + os.environ["WORKDIR"] = os.environ["RVIC_TEST_DIR"] + print('Setting to output run_tests.py dir to $WORKDIR: ' + '{0}\n'.format(os.environ["WORKDIR"])) + # -------------------------------------------------------------------- # def main(): @@ -55,7 +68,15 @@ def run_examples(config_file): print("".center(100, '-')) print("Starting Test #{0} of {1}: {2}".format(i+1, num_tests, test).center(100)) + desc = textwrap.fill(", ".join(test_dict['description']), 100) + print("Description: {0}".format(desc)) print("".center(100, '-')) + + if 'processors' in test_dict: + numofproc = test_dict['processors'] + else: + numofproc = 1 + pr = cProfile.Profile() pr.enable() @@ -64,7 +85,8 @@ def run_examples(config_file): elif test_dict['function'] == 'convolution': convolution.convolution(test_dict['config_file']) elif test_dict['function'] == 'parameters': - parameters.parameters(test_dict['config_file']) + parameters.parameters(test_dict['config_file'], + numofproc=numofproc) else: raise ValueError('Unknow function variable: ' '{0}'.format(test_dict['function'])) From c2ab5b6c527841b3141b3db73e0d55f045bbd3e6 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Fri, 18 Apr 2014 11:36:03 -0700 Subject: [PATCH 11/13] change dictionary that read_config returns to an OrderedDict --- rvic/core/config.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rvic/core/config.py b/rvic/core/config.py index 191d645..31cbf99 100644 --- a/rvic/core/config.py +++ b/rvic/core/config.py @@ -4,6 +4,7 @@ """ import os +from collections import OrderedDict from ConfigParser import SafeConfigParser @@ -34,10 +35,10 @@ def read_config(config_file): config.optionxform = str config.read(config_file) sections = config.sections() - dict1 = {} + dict1 = OrderedDict() for section in sections: options = config.options(section) - dict2 = {} + dict2 = OrderedDict() for option in options: dict2[option] = config_type(config.get(section, option)) dict1[section] = dict2 From 84b6f0798b8bc4a89fc287b32afa80767aac4f6b Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Fri, 18 Apr 2014 11:37:18 -0700 Subject: [PATCH 12/13] fix to history module that was keeping single timestep files from being written --- rvic/convolution.py | 9 +++--- rvic/core/history.py | 66 ++++++++++++++++++++++++++++---------------- 2 files changed, 47 insertions(+), 28 deletions(-) diff --git a/rvic/convolution.py b/rvic/convolution.py index 173dade..3c1022b 100644 --- a/rvic/convolution.py +++ b/rvic/convolution.py @@ -14,6 +14,7 @@ Major updates to the... """ import os +from collections import OrderedDict from logging import getLogger from core.log import init_logger, close_logger, LOG_NAME from core.utilities import make_directories, read_domain @@ -168,7 +169,7 @@ def convolution_init(config_file): # Setup history Tape(s) and Write Initial Outputs history = config_dict['HISTORY'] numtapes = int(history['RVICHIST_NTAPES']) - hist_tapes = {} + hist_tapes = OrderedDict() # make sure history file fields are all in list form if numtapes == 1: @@ -185,14 +186,14 @@ def convolution_init(config_file): RvicDomainFile=os.path.split(domain['FILE_NAME'])[1]) for j in xrange(numtapes): - tapename = 'Tape.%i' % j + tapename = 'Tape.{0}'.format(j) log.info('setting up History %s', tapename) hist_tapes[tapename] = Tape(time_handle.time_ord, options['CASEID'], rout_var, tape_num=j, fincl=['streamflow'], - mfilt=int(history['RVICHIST_MFILT'][j]), + mfilt=history['RVICHIST_MFILT'][j], ndens=int(history['RVICHIST_NDENS'][j]), nhtfrq=int(history['RVICHIST_NHTFRQ'][j]), avgflag=history['RVICHIST_AVGFLAG'][j], @@ -220,7 +221,7 @@ def convolution_init(config_file): # -------------------------------------------------------------------- # def convolution_run(hist_tapes, data_model, rout_var, dom_data, time_handle, - directories, config_dict): + directories, config_dict): """ Main run loop for RVIC model. """ diff --git a/rvic/core/history.py b/rvic/core/history.py index 6bc89e7..fbbb345 100644 --- a/rvic/core/history.py +++ b/rvic/core/history.py @@ -8,9 +8,12 @@ - initialization: sets tape options, determines filenames, etc. - update: method that incorporates new fluxes into the history tape. - __next_update_out_data: method to determine when to update the - outdata container - + out_data container + - __next_write_out_data: method to determine when to write the out_data + container + - finish: method to close all remaining history tapes. """ + import os import numpy as np from netCDF4 import Dataset, date2num, num2date, stringtochar @@ -46,7 +49,7 @@ def __init__(self, time_ord, caseid, Rvar, tape_num=0, self._tape_num = tape_num self._time_ord = time_ord # Days since basetime self._caseid = caseid # Case ID and prefix for outfiles - self._fincl = fincl # Fields to include in history file + self._fincl = list(fincl) # Fields to include in history file self._mfilt = mfilt # Maximum number of time samples self._ndens = ndens if self._ndens == 1: # Output file precision @@ -78,6 +81,7 @@ def __init__(self, time_ord, caseid, Rvar, tape_num=0, else: # If monthly self._out_data_stepsize = None # varies by month + log.debug('_out_data_stepsize: ', self._out_data_stepsize) # ------------------------------------------------------------ # # ------------------------------------------------------------ # @@ -91,7 +95,7 @@ def __init__(self, time_ord, caseid, Rvar, tape_num=0, raise ValueError('Must include grid lons / lats if ' 'outtype == grid') else: - self._out_data_shape = self._num_outlets + self._out_data_shape = (self._num_outlets, ) # ------------------------------------------------------------ # # ------------------------------------------------------------ # @@ -181,6 +185,8 @@ def __init__(self, time_ord, caseid, Rvar, tape_num=0, # Determine when the update of out_data should be self.__next_update_out_data() # ------------------------------------------------------------ # + + log.debug(self.__repr__()) # ---------------------------------------------------------------- # # ---------------------------------------------------------------- # @@ -190,18 +196,18 @@ def __str__(self): def __repr__(self): parts = ['------- Summary of History Tape Settings -------', - '\t# caseid: {0}s'.format(self._caseid), - '\t# fincl: {0}s'.format(','.join(self._fincl)), - '\t# nhtfrq: {0}s'.format(self._nhtfrq), - '\t# mfilt: {0}s'.format(self._mfilt), - '\t# ncprec: {0}s'.format(self._ncprec), - '\t# avgflag: {0}s'.format(self._avgflag), - '\t# fname_format: {0}s'.format(self._fname_format), - '\t# file_format: {0}s'.format(self._file_format), - '\t# outtype: {0}s'.format(self._outtype), - '\t# out_dir: {0}s'.format(self._out_dir), - '\t# calendar: {0}s'.format(self._calendar), - '\t# units: {0}s'.format(self._units), + '\t# caseid: {0}'.format(self._caseid), + '\t# fincl: {0}'.format(','.join(self._fincl)), + '\t# nhtfrq: {0}'.format(self._nhtfrq), + '\t# mfilt: {0}'.format(self._mfilt), + '\t# ncprec: {0}'.format(self._ncprec), + '\t# avgflag: {0}'.format(self._avgflag), + '\t# fname_format: {0}'.format(self._fname_format), + '\t# file_format: {0}'.format(self._file_format), + '\t# outtype: {0}'.format(self._outtype), + '\t# out_dir: {0}'.format(self._out_dir), + '\t# calendar: {0}'.format(self._calendar), + '\t# units: {0}'.format(self._units), ' ------- End of History Tape Settings -------'] return '\n'.join(parts) # ---------------------------------------------------------------- # @@ -276,7 +282,10 @@ def write_initial(self): # ---------------------------------------------------------------- # def __next_write_out_data(self): - """ """ + """determine the maximum size of out_data""" + + log.debug('determining size of out_data') + self._out_data_i = 0 # position counter for out_data array # ------------------------------------------------------------ # @@ -297,7 +306,6 @@ def __next_write_out_data(self): # calculate the mfilt value mfilt = int(round((b1 - b0) / self._out_data_stepsize)) - elif self._mfilt == 'month': if self._nhtfrq == 0: mfilt = 1 @@ -311,19 +319,19 @@ def __next_write_out_data(self): # calculate the mfilt value mfilt = int(round((b1 - b0) / self._out_data_stepsize)) - elif self._mfilt == 'day': - if self._nhtfrq == 0: + if self._nhtfrq != 0: + b1 = b0 + 1.0 + else: raise ValueError('Incompatable values for NHTFRQ and MFILT') - b1 = b0 + 1.0 # calculate the mfilt value mfilt = int(round((b1 - b0) / self._out_data_stepsize)) - else: - mfilt = self._mfilt + mfilt = int(self._mfilt) # ------------------------------------------------------------ # + # ------------------------------------------------------------ # if mfilt < 1: mfilt = 1 @@ -334,14 +342,21 @@ def __next_write_out_data(self): shape = (mfilt, ) + self._out_data_shape + log.debug('out_data shape: %s', shape) + log.debug('_out_data_write: %s', self._out_data_write) + for field in self._fincl: self._out_data[field] = np.zeros(shape, dtype=np.float64) + + self._out_data_has_values = False # ---------------------------------------------------------------- # # ---------------------------------------------------------------- # # fill in out_data def __update_out_data(self): + self._out_data_has_values = True + # ------------------------------------------------------------ # # Update the _out_data fields for field in self._fincl: @@ -368,12 +383,15 @@ def __update_out_data(self): self._out_data_i = 0 else: self._out_data_i += 1 + log.debug('out_data counter is %s of %s', self._out_data_i, + self._out_data_write) # ---------------------------------------------------------------- # # ---------------------------------------------------------------- # def finish(self): """write out_data""" - if self._out_data_i > 0: + log.debug('finishing tape %s', self._tape_num) + if self._out_data_has_values: if self._outtype == 'grid': self.__write_grid() else: From 02334d5d4687fc63e6d9d1fd1168fac39909dbcf Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Fri, 18 Apr 2014 12:21:36 -0700 Subject: [PATCH 13/13] fixes for irregular grid handling of coordinate variables --- rvic/core/read_forcing.py | 11 +++++++---- rvic/core/utilities.py | 6 ++++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/rvic/core/read_forcing.py b/rvic/core/read_forcing.py index e7307e8..84572c3 100644 --- a/rvic/core/read_forcing.py +++ b/rvic/core/read_forcing.py @@ -116,10 +116,13 @@ def __init__(self, path, file_str, time_fld, lat_fld, liq_flds, # determine the latitude order lats = f.variables[self.lat_fld][:] - if lats[-1] > lats[0]: - log.debug('Input fluxes came in upside down, flipping ' - 'params and maybe domain.') - self.lat0_is_min = True + if lats.ndim == 1: + if lats[-1] > lats[0]: + log.debug('Input fluxes came in upside down, flipping ' + 'params and maybe domain.') + self.lat0_is_min = True + else: + self.lat0_is_min = False else: self.lat0_is_min = False else: diff --git a/rvic/core/utilities.py b/rvic/core/utilities.py index 3817dc0..6ab3dfe 100644 --- a/rvic/core/utilities.py +++ b/rvic/core/utilities.py @@ -284,9 +284,11 @@ def read_domain(domain_dict, lat0_is_min=False): # Make sure the longitude / latitude vars are 2d dom_lat = domain_dict['LATITUDE_VAR'] dom_lon = domain_dict['LONGITUDE_VAR'] + + dom_data['cord_lons'] = dom_data[dom_lon][:] + dom_data['cord_lats'] = dom_data[dom_lat][:] + if dom_data[dom_lon].ndim == 1: - dom_data['cord_lons'] = dom_data[dom_lon][:] - dom_data['cord_lats'] = dom_data[dom_lat][:] # ------------------------------------------------------------- # # Check latitude order, flip if necessary. if (dom_data[dom_lat][-1] > dom_data[dom_lat][0]) != lat0_is_min: