Merge branch 'master' into master

README.rst

@@ -1,12 +1,17 @@
-Nifty TEST
-==========
-
-.. image:: https://zenodo.org/badge/93109208.svg
-   :alt: DOI of the latest release. See releases.
-   :target: https://zenodo.org/record/852696#.WaWmr5PyhMA
+Nifty
+=====
+.. image:: https://badge.fury.io/py/nifty4gemini.svg
+    :alt: Available on PyPi.
+    :target: https://badge.fury.io/py/nifty4gemini
 .. image:: https://readthedocs.org/projects/nifty4gemini/badge/?version=latest
    :alt: Nifty's documentation, hosted on ReadtheDocs.
    :target: http://nifty4gemini.readthedocs.io/en/latest/
+.. image:: https://zenodo.org/badge/93109208.svg
+   :alt: DOI of the latest release. See releases.
+   :target: https://zenodo.org/record/852696#.WaWmr5PyhMA
+.. image:: https://img.shields.io/badge/License-MIT-yellow.svg
+   :alt: MIT license.
+   :target: https://opensource.org/licenses/MIT
 .. image:: http://img.shields.io/badge/powered%20by-AstroPy-orange.svg?style=flat
    :alt: Nifty uses Astropy! Here is a link to the project webpage:
    :target: http://www.astropy.org/

build/lib/nifty/pipeline/downloadFromGeminiPublicArchive/ndmapperDownloader.py

@@ -38,7 +38,7 @@
 import tarfile
 import hashlib
 
-def download_query_gemini(query, dirname=''):
+def download_query_gemini(query, dirname='', cookieName=''):
     """
     Perform a user-specified Gemini science archive query and save the files
     returned to a specified directory.
@@ -67,10 +67,18 @@ def download_query_gemini(query, dirname=''):
     checksum_fn = 'md5sums.txt'
     aux_fn = [checksum_fn, 'README.txt']
 
+
     # Perform Web query and download the tar file to a StringIO file object
     # in memory, passing through any HTTP errors.
-    with closing(urllib2.urlopen(query)) as fileobj:
-        fobj_buff = StringIO(fileobj.read())
+    # Added by ncomeau: support for proprietary downloads
+    if cookieName:
+        opener = urllib2.build_opener()
+        opener.addheaders.append(('Cookie', 'gemini_archive_session={}'.format(cookieName)))
+        with closing(opener.open(query)) as fileobj:
+            fobj_buff = StringIO(fileobj.read())
+    else:
+        with closing(urllib2.urlopen(query)) as fileobj:
+            fobj_buff = StringIO(fileobj.read())
 
     # Open the in-memory tar file & extract its contents.
     with tarfile.open(fileobj=fobj_buff) as tar_obj:

build/lib/nifty/pipeline/nifsPipeline.py

@@ -73,7 +73,7 @@
 #                                      +
 #
 
-# Welcome to Nifty.
+# Welcome to Nifty!
 
 # The current version:
 # TODO(nat): fix this to import the version from setup.py.
@@ -214,7 +214,6 @@ def start(args):
         if manualMode:
             a = raw_input('About to enter nifsReduce to reduce science.')
         nifsReduce.start('Science')
-
     if telluricCorrection:
         if manualMode:
             a = raw_input('About to enter nifsTelluric to make and create telluric corrected cubes.')

build/lib/nifty/pipeline/objectoriented/getConfig.py

@@ -101,18 +101,6 @@ def makeConfig(self):
                 self.config.write(self.outfile)
             logging.info("\nData reduction parameters for this reduction were copied from recipes/defaultConfig.cfg to ./config.cfg.")
 
-        if self.repeat:
-            logging.info("\nOverwriting ./config.cfg with saved config from most recent data reduction.")
-            if os.path.exists('./' + self.configFile):
-                os.remove('./' + self.configFile)
-            shutil.copy(self.RUNTIME_DATA_PATH + self.configFile, './' + self.configFile)
-
-        # Print data reduction parameters for a user's peace-of-mind.
-        logging.info("\nSaving data reduction parameters.")
-        if os.path.exists(self.RUNTIME_DATA_PATH + self.configFile):
-            os.remove(self.RUNTIME_DATA_PATH + self.configFile)
-        shutil.copy('./' + self.configFile, self.RUNTIME_DATA_PATH + self.configFile)
-
         # TODO(nat): fix this. It isn't recursively printing the dictionaries of values.
         logging.info("\nParameters for this data reduction as read from ./config.cfg:\n")
         with open(self.configFile) as self.config_file:

build/lib/nifty/pipeline/steps/nifsFluxCalibrate.py

@@ -18,14 +18,15 @@ def run():
     """
     # Store current working directory for later use.
     path = os.getcwd()
-
+    import pdb; pdb.set_trace()
+    # Set up the logging file.
+    log = os.getcwd()+'/Nifty.log'
     # Set up iraf
     iraf.gemini()
-    iraf.unlearn("gemini")
+    #iraf.unlearn("gemini")
+
     #iraf.unlearn(iraf.gemini,iraf.gemtools,iraf.gnirs,iraf.nifs,iraf.imcopy)
 
-    # Set up the logging file.
-    log = os.getcwd()+'/Nifty.log'
 
     logging.info('\n#################################################')
     logging.info('#                                               #')
@@ -305,13 +306,25 @@ def scaleBlackBody(rawFrame, log, over):
     if os.path.exists("5_scaledBBody"+rawFrame+".fits"):
         if over:
             os.remove("5_scaledBBody"+rawFrame+".fits")
-            iraf.imarith(operand1="3_BBody"+rawFrame, op="*", operand2=bbodyScaleFactor, result="5_scaledBBody"+rawFrame,title='',divzero=0.0,hparams='',pixtype='',calctype='',verbose='no',noact='no',mode='al')
+            # A bug involving iraf.gemini() causes imarith to fail here. Use astropy unless you fixed it.
+            #iraf.imarith(operand1="3_BBody"+rawFrame, op="*", operand2=bbodyScaleFactor, result="5_scaledBBody"+rawFrame,title='',divzero=0.0,hparams='',pixtype='',calctype='',verbose='no',noact='no',mode='al')
+            operand1 = astropy.io.fits.open("3_BBody"+rawFrame+".fits")[0].data
+            operand2 = bbodyScaleFactor
+            multiplied = operand1 * operand2
+            hdu = astropy.io.fits.PrimaryHDU(multiplied)
+            hdu.writeto("5_scaledBBody"+rawFrame+".fits")
+
             logging.info("\nCreated a scaled blackbody, 5_scaledBBody{}.fits".format(rawFrame))
         else:
             logging.info("\nOutput exists and -over not set - skipping production of scaled black body")
     else:
-        iraf.imarith(operand1="3_BBody"+rawFrame, op="*", operand2=bbodyScaleFactor, result="5_scaledBBody"+rawFrame,title='',divzero=0.0,hparams='',pixtype='',calctype='',verbose='no',noact='no',mode='al')
-        logging.info("\nCreated a scaled blackbody, 5_scaledBBody{}.fits".format(rawFrame))
+        # A bug involving iraf.gemini() causes imarith to fail here. Use astropy unless you fixed it.
+        #iraf.imarith(operand1="3_BBody"+rawFrame, op="*", operand2=bbodyScaleFactor, result="5_scaledBBody"+rawFrame,title='',divzero=0.0,hparams='',pixtype='',calctype='',verbose='no',noact='no',mode='al')
+        operand1 = astropy.io.fits.open("3_BBody"+rawFrame+".fits")[0].data
+        operand2 = bbodyScaleFactor
+        multiplied = operand1 * operand2
+        hdu = astropy.io.fits.PrimaryHDU(multiplied)
+        hdu.writeto("5_scaledBBody"+rawFrame+".fits")
     # We now have a scaled blackbody, scaledBlackBody.fits
 
 def multiplyByBlackBody(rawFrame, log, over):

build/lib/nifty/pipeline/steps/nifsSort.py

@@ -127,6 +127,7 @@ def start():
         sortConfig = options['sortConfig']
         rawPath = sortConfig['rawPath']
         program = sortConfig['program']
+        proprietaryCookie = sortConfig['proprietaryCookie']
         skyThreshold = sortConfig['skyThreshold']
         sortTellurics = sortConfig['sortTellurics']
         telluricTimeThreshold = sortConfig['telluricTimeThreshold']
@@ -145,7 +146,10 @@ def start():
             os.mkdir('./rawData')
         logging.info('\nDownloading data from Gemini public archive to ./rawData. This will take a few minutes.')
         logging.info('\nURL used for the download: \n' + str(url))
-        download_query_gemini(url, './rawData')
+        if proprietaryCookie:
+            download_query_gemini(url, './rawData', proprietaryCookie)
+        else:
+            download_query_gemini(url, './rawData')
         rawPath = os.getcwd()+'/rawData'
 
 
@@ -1402,7 +1406,7 @@ def matchTellurics(telDirList, obsDirList, telluricTimeThreshold):
                 logging.info("#####################################################################")
                 logging.info("#####################################################################\n")
 
-                rewriteSciImageList(2.0, Science)
+                rewriteSciImageList(2.0, "Science")
                 try:
                     sciImageList = open('scienceFrameList', "r").readlines()
                     logging.info("\nSucceeded; a science frame list exists in " + str(os.getcwd()))

build/lib/nifty/pipeline/steps/nifsTelluric.py

@@ -51,6 +51,20 @@ def run():
         standardStarRA = telluricCorrectionConfig['standardStarRA']
         standardStarDec = telluricCorrectionConfig['standardStarDec']
 
+    # TESTING IRAF
+    iraf.gemini()
+    #iraf.gemtools()
+    #iraf.gnirs()
+    #iraf.nifs()
+
+    #iraf.unlearn(iraf.gemini,iraf.gemtools,iraf.gnirs,iraf.nifs)
+
+    #iraf.set(stdimage='imt2048')
+    #iraf.nsheaders("nifs", logfile=log)
+    #user_clobber=iraf.envget("clobber")
+    #iraf.reset(clobber='yes')
+
+
     for scienceDirectory in scienceDirectoryList:
         try:
             os.chdir(scienceDirectory + '/products_telluric_corrected')
@@ -478,18 +492,44 @@ def divideByContinuum(rawFrame, log, over):
     if os.path.exists("3_chtel"+rawFrame+'.fits'):
         if over:
             os.remove("3_chtel"+rawFrame+'.fits')
-            iraf.imarith("1_htel"+rawFrame+'.fits', "/", "2_fit"+rawFrame+'.fits', result="3_chtel"+rawFrame+'.fits',title='',divzero=0.0,hparams='',pixtype='',calctype='',verbose='no',noact='no',mode='al')
+            #iraf.imarith("1_htel"+rawFrame+'.fits', "/", "2_fit"+rawFrame+'.fits', result="3_chtel"+rawFrame+'.fits',title='',divzero=0.0,hparams='',pixtype='',calctype='',verbose='no',noact='no',mode='al')
+            operand1 = astropy.io.fits.open("1_htel"+rawFrame+'.fits')[0].data
+            operand2 = astropy.io.fits.open("2_fit"+rawFrame+'.fits')[0].data
+            header = astropy.io.fits.open("1_htel"+rawFrame+'.fits')[0].header
+            multiplied = np.array(operand1, copy=True)
+            for i in range(len(multiplied)):
+                if operand2[i] != 0:
+                    multiplied[i] = operand1[i] / operand2[i]
+                else:
+                    multiplied[i] = 0.0
+            hdu = astropy.io.fits.PrimaryHDU(multiplied)
+            hdu.header = header
+            hdu.writeto("3_chtel"+rawFrame+".fits")
+            logging.info("\nDivided telluric correction by continuum")
         else:
             logging.info("\nOutput exists and -over not set - skipping division by continuum")
     else:
-        iraf.imarith('1_htel'+rawFrame+'.fits', "/", '2_fit'+rawFrame+'.fits', result='3_chtel'+rawFrame+'.fits',title='',divzero=0.0,hparams='',pixtype='',calctype='',verbose='no',noact='no',mode='al')
+        #iraf.imarith('1_htel'+rawFrame+'.fits', "/", '2_fit'+rawFrame+'.fits', result='3_chtel'+rawFrame+'.fits',title='',divzero=0.0,hparams='',pixtype='',calctype='',verbose='no',noact='no',mode='al')
+        operand1 = astropy.io.fits.open("1_htel"+rawFrame+'.fits')[0].data
+        operand2 = astropy.io.fits.open("2_fit"+rawFrame+'.fits')[0].data
+        header = astropy.io.fits.open("1_htel"+rawFrame+'.fits')[0].header
+        multiplied = np.array(operand1, copy=True)
+        for i in range(len(multiplied)):
+            if operand2[i] != 0:
+                multiplied[i] = operand1[i] / operand2[i]
+            else:
+                multiplied[i] = 0.0
+        hdu = astropy.io.fits.PrimaryHDU(multiplied)
+        hdu.header = header
+        hdu.writeto("3_chtel"+rawFrame+".fits")
+        logging.info("\nDivided telluric correction by continuum")
 
 def get1dSpecFromCube(rawFrame, log, over):
     """
     Turn a cube into a 1D spec, used to find shift and scale values of telluric spectrum.
     Currently: Extracts 1D spectra from center of cube.
     """
-    cube = astropy.io.fits.open('../products_uncorrected/ctfbrsn'+rawFrame+'.fits')
+    cube = astropy.io.fits.open('ctfbrsn'+rawFrame+'.fits')
     cubeheader = cube[1].header
     cubeslice = cube[1].data[:,30,30]
     # Create a PrimaryHDU object to encapsulate the data and header.
@@ -684,12 +724,41 @@ def vega(rawFrame, grating, hLineInter, log, over):
 
     if os.path.exists("final_tel_no_hLines_no_norm.fits"):
         if over:
-            os.remove("final_tel_no_hLines_no_norm.fits")
-            iraf.imarith(operand1="1_htel" + rawFrame, op='/', operand2=norm, result='final_tel_no_hLines_no_norm', title='', divzero=0.0, hparams='', pixtype='', calctype='', verbose='yes', noact='no', mode='al')
+            # Subtle bugs in iraf mean imarith doesn't work. So we use an astropy/numpy solution.
+            # Open the image and the scalar we will be dividing it by.
+            operand1 = astropy.io.fits.open("1_htel" + rawFrame+'.fits')[0].data
+            operand2 = float(norm)
+            # Create a new data array
+            multiplied = np.array(operand1, copy=True)
+            # Don't forget to include the original header! If you don't later IRAF tasks get confused.
+            header = astropy.io.fits.open("1_htel" + rawFrame+'.fits')[0].header
+            for i in range(len(multiplied)):
+                if operand2 != 0:
+                    multiplied[i] = operand1[i] / operand2
+                else:
+                    multiplied[i] = 1
+            # Set the data and header of the in-memory image
+            hdu = astropy.io.fits.PrimaryHDU(multiplied)
+            hdu.header = header
+            # Finally, write the new image to a new .fits file. It only has one extension; zero, with a header and data.
+            hdu.writeto('final_tel_no_hLines_no_norm.fits')
+            #iraf.imarith(operand1="1_htel" + rawFrame, op='/', operand2=norm, result='final_tel_no_hLines_no_norm', title='', divzero=0.0, hparams='', pixtype='', calctype='', verbose='yes', noact='no', mode='al')
         else:
             logging.info("Output file exists and -over not set - skipping H line normalization correction")
     else:
-        iraf.imarith(operand1="1_htel" + rawFrame, op='/', operand2=norm, result='final_tel_no_hLines_no_norm', title='', divzero=0.0, hparams='', pixtype='', calctype='', verbose='yes', noact='no', mode='al')
+        #iraf.imarith(operand1="1_htel" + rawFrame, op='/', operand2=norm, result='final_tel_no_hLines_no_norm', title='', divzero=0.0, hparams='', pixtype='', calctype='', verbose='yes', noact='no', mode='al')
+        operand1 = astropy.io.fits.open("1_htel" + rawFrame+'.fits')[0].data
+        operand2 = float(norm)
+        multiplied = np.array(operand1, copy=True)
+        header = astropy.io.fits.open("1_htel" + rawFrame+'.fits')[0].header
+        for i in range(len(multiplied)):
+            if operand2 != 0:
+                multiplied[i] = operand1[i] / operand2
+            else:
+                multiplied[i] = 1
+        hdu = astropy.io.fits.PrimaryHDU(multiplied)
+        hdu.header = header
+        hdu.writeto('final_tel_no_hLines_no_norm.fits')
 
     if os.path.exists('final_tel_no_hLines_no_norm.fits'):
         os.remove("1_htel" + rawFrame + ".fits")

docs/index.rst

@@ -8,6 +8,8 @@ Welcome to Nifty's documentation!
 
 Nifty is a full-featured Gemini NIFS data reduction pipeline and data reduction package.
 
+This is a test.
+
 =====================
 Package Documentation
 =====================