.bumpversion.cfg

 [bumpversion]
-current_version = 1.3.0
+current_version = 1.3.1
 commit = True
 tag = True
 

.pre-commit-config.yaml

+exclude: '^$'
+fail_fast: false
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.2.3
+    hooks:
+    - id: flake8
+      additional_dependencies: [flake8-docstrings, flake8-debugger, flake8-bugbear]

CHANGELOG.md

+## Version v1.3.1 (2020/02/07)
+
+### Issues Closed
+
+* [Issue 52](https://github.com/pytroll/pygac/issues/52) - Allow gzip input files ([PR 53](https://github.com/pytroll/pygac/pull/53))
+* [Issue 49](https://github.com/pytroll/pygac/issues/49) - Calibration Coeffs Patmos-X 2017 ([PR 50](https://github.com/pytroll/pygac/pull/50))
+
+In this release 2 issues were closed.
+
+### Pull Requests Merged
+
+#### Bugs fixed
+
+* [PR 50](https://github.com/pytroll/pygac/pull/50) - Fix typo in MetOp-B calibration ([49](https://github.com/pytroll/pygac/issues/49))
+* [PR 48](https://github.com/pytroll/pygac/pull/48) - Update metadata when reading lonlat also
+
+#### Features added
+
+* [PR 53](https://github.com/pytroll/pygac/pull/53) - Allow gzip compressed files as input for klm and pod readers read method ([52](https://github.com/pytroll/pygac/issues/52))
+* [PR 51](https://github.com/pytroll/pygac/pull/51) - Use the data format name to id ARS headered files
+
+In this release 4 pull requests were closed.
+
+
 ## Version 1.3.0 (2019/12/05)
 
 

debian/changelog

-pygac (1.3.0-2) UNRELEASED; urgency=medium
+pygac (1.3.1-1) unstable; urgency=medium
 
+  * New upstream release.
   * Bump Standards-Version to 4.5.0, no changes.
+  * Update debian/copyright.
+  * debian/patches:
+    - drop 0001-Temporary-disable-broken-tests.patch,
+      no longer necessary
+    - refresh and renumber remaining patches
 
- -- Antonio Valentino <antonio.valentino@tiscali.it>  Sat, 25 Jan 2020 11:01:54 +0100
+ -- Antonio Valentino <antonio.valentino@tiscali.it>  Sat, 08 Feb 2020 08:22:06 +0000
 
 pygac (1.3.0-1) unstable; urgency=medium
 

debian/copyright

@@ -12,6 +12,7 @@ Copyright: 2010-2019 Martin Raspaud <martin.raspaud@smhi.se>
                      Cornelia Schlundt <cornelia.schlundt@dwd.de>
                      Sajid Pareeth <sajid.pareeth@fmach.it>
                      Nina Hakansson <nina.hakansson@smhi.se>
+                     Carlos Horn <carlos.horn@external.eumetsat.int>
                      Pytroll Developers
 License: GPL-3+
 

debian/patches/0001-Temporary-disable-broken-tests.patch

-From: Antonio Valentino <antonio.valentino@tiscali.it>
-Date: Mon, 24 Dec 2018 08:16:01 +0000
-Subject: Temporary disable broken tests
-
-See also upstream issue https://github.com/pytroll/pygac/issues/16.
----
- pygac/tests/test_calibrate_klm.py | 1 +
- pygac/tests/test_calibrate_pod.py | 1 +
- 2 files changed, 2 insertions(+)
-
-diff --git a/pygac/tests/test_calibrate_klm.py b/pygac/tests/test_calibrate_klm.py
-index 8a489ce..1f2abd3 100644
---- a/pygac/tests/test_calibrate_klm.py
-+++ b/pygac/tests/test_calibrate_klm.py
-@@ -33,6 +33,7 @@ from pygac.calibration import calibrate_solar, calibrate_thermal
- 
- class TestGenericCalibration(unittest.TestCase):
- 
-+    @unittest.skip('temorary disabled, see upstream gh-16')
-     def test_calibration_vis(self):
- 
-         counts = np.array([[0, 0, 0, 0, 0,
-diff --git a/pygac/tests/test_calibrate_pod.py b/pygac/tests/test_calibrate_pod.py
-index fe1bffc..aa35d11 100644
---- a/pygac/tests/test_calibrate_pod.py
-+++ b/pygac/tests/test_calibrate_pod.py
-@@ -33,6 +33,7 @@ from pygac.calibration import calibrate_solar, calibrate_thermal
- 
- class TestGenericCalibration(unittest.TestCase):
- 
-+    @unittest.skip('temorary disabled, see upstream gh-16')
-     def test_calibration_vis(self):
- 
-         counts = np.array([[0, 0, 0, 0, 0,

debian/patches/series

-0001-Temporary-disable-broken-tests.patch
-0002-Fix-config.patch
+0001-Fix-config.patch

pygac/calibration.py

@@ -29,8 +29,8 @@ import numpy as np
 coeffs = {
     'metopb': {'ah': np.array([0.166, 0.183, 0.201]),
                'al': np.array([0.055, 0.061, 0.029]),
-               'bh': np.array([2.019, 1.476, 1.478]),
-               'bl': np.array([2.019, 1.476, 1.478]),
+               'bh': np.array([2.019, 1.476, 1.748]),
+               'bl': np.array([2.019, 1.476, 1.748]),
                'ch': np.array([-0.201, -0.137, -0.033]),
                'cl': np.array([-0.201, -0.137, -0.033]),
                'c_dark': np.array([39.70, 40.00, 40.30]),
@@ -405,6 +405,9 @@ coeffs = {
                'b1': np.array([1 - 0.0, 1 - 0.11187, 1 - 0.05991]),
                'b2': np.array([0.0, 0.00054668, 0.00024985])}}
 
+"""Source Patmos-X Coeffs: Version Tag 'y2017r1_sbaf'
+https://cimss.ssec.wisc.edu/patmosx/avhrr_cal.html"""
+
 
 class Calibrator(object):
 

pygac/klm_reader.py

@@ -9,6 +9,7 @@
 #   Sajid Pareeth <sajid.pareeth@fmach.it>
 #   Martin Raspaud <martin.raspaud@smhi.se>
 #   Adam Dybbroe <adam.dybbroe@smhi.se>
+#   Carlos Horn <carlos.horn@external.eumetsat.int>
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -587,28 +588,39 @@ class KLMReader(Reader):
                 The scanlines
 
         """
-        super(KLMReader, self).read(filename=filename)
-        with open(filename) as fd_:
-            self.ars_head = np.fromfile(fd_, dtype=ars_header, count=1)[0]
-            if not self.ars_head['data_set_name'].startswith(self.creation_site):
+        with self._open(filename) as fd_:
+            # Note that np.fromfile does not work with gzip.GzipFile
+            # objects (numpy version 1.16.4), because it restricts the
+            # file objects to (io.FileIO, io.BufferedReader, io.BufferedWriter)
+            # see: numpy.compat.py3k.isfileobj
+            self.ars_head, = np.frombuffer(
+                fd_.read(ars_header.itemsize),
+                dtype=ars_header, count=1)
+            if not self.ars_head['data_format'].startswith(b'NOAA Level 1b'):
                 fd_.seek(0)
                 self.ars_head = None
                 ars_offset = 0
             else:
                 ars_offset = ars_header.itemsize
-            self.head = np.fromfile(fd_, dtype=header, count=1)[0]
+            self.head, = np.frombuffer(
+                fd_.read(header.itemsize),
+                dtype=header, count=1)
             self.header_version = self.head[
                 "noaa_level_1b_format_version_number"]
             if self.header_version >= 5:
-                self.analog_telemetry = np.fromfile(
-                    fd_, dtype=analog_telemetry_v5, count=1)[0]
+                self.analog_telemetry, = np.frombuffer(
+                    fd_.read(analog_telemetry_v5.itemsize),
+                    dtype=analog_telemetry_v5, count=1)
             else:
-                self.analog_telemetry = np.fromfile(
-                    fd_, dtype=analog_telemetry_v2, count=1)[0]
+                self.analog_telemetry, = np.frombuffer(
+                    fd_.read(analog_telemetry_v2.itemsize),
+                    dtype=analog_telemetry_v2, count=1)
             # LAC: 1, GAC: 2, ...
             self.data_type = self.head['data_type_code']
             fd_.seek(self.offset + ars_offset, 0)
-            self.scans = np.fromfile(fd_, dtype=self.scanline_type,
+            self.scans = np.frombuffer(
+                fd_.read(),
+                dtype=self.scanline_type,
                 count=self.head["count_of_data_records"])
 
         self.correct_scan_line_numbers()

pygac/pod_reader.py

@@ -8,6 +8,7 @@
 #   Adam Dybbroe <adam.dybbroe@smhi.se>
 #   Sajid Pareeth <sajid.pareeth@fmach.it>
 #   Martin Raspaud <martin.raspaud@smhi.se>
+#   Carlos Horn <carlos.horn@external.eumetsat.int>
 
 # This work was done in the framework of ESA-CCI-Clouds phase I
 
@@ -216,11 +217,12 @@ class PODReader(Reader):
                 The scanlines
 
         """
-        super(PODReader, self).read(filename=filename)
         # choose the right header depending on the date
-        with open(filename) as fd_:
+        with self._open(filename) as fd_:
             # read archive header
-            self.tbm_head = np.fromfile(fd_, dtype=tbm_header, count=1)[0]
+            self.tbm_head, = np.frombuffer(
+                fd_.read(tbm_header.itemsize),
+                dtype=tbm_header, count=1)
             if ((not self.tbm_head['data_set_name'].startswith(self.creation_site + b'.')) and
                     (self.tbm_head['data_set_name'] != b'\x00' * 42 + b'  ')):
                 fd_.seek(0)
@@ -229,7 +231,9 @@ class PODReader(Reader):
             else:
                 tbm_offset = tbm_header.itemsize
 
-            head = np.fromfile(fd_, dtype=header0, count=1)[0]
+            head, = np.frombuffer(
+                fd_.read(header0.itemsize),
+                dtype=header0, count=1)
             year, jday, _ = self.decode_timestamps(head["start_time"])
 
             start_date = (datetime.date(year, 1, 1) +
@@ -243,9 +247,12 @@ class PODReader(Reader):
                 header = header3
 
             fd_.seek(tbm_offset, 0)
-            self.head = np.fromfile(fd_, dtype=header, count=1)[0]
+            self.head, = np.frombuffer(
+                fd_.read(header.itemsize),
+                dtype=header, count=1)
             fd_.seek(self.offset + tbm_offset, 0)
-            self.scans = np.fromfile(fd_,
+            self.scans = np.frombuffer(
+                fd_.read(),
                 dtype=self.scanline_type,
                 count=self.head["number_of_scans"])
 

pygac/reader.py

@@ -6,6 +6,7 @@
 # Author(s):
 
 #   Martin Raspaud <martin.raspaud@smhi.se>
+#   Carlos Horn <carlos.horn@external.eumetsat.int>
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -30,6 +31,8 @@ import numpy as np
 import os
 import six
 import types
+import gzip
+from contextlib import contextmanager
 
 from pygac import (CONFIG_FILE, centered_modulus,
                    calculate_sun_earth_distance_correction,
@@ -120,8 +123,31 @@ class Reader(six.with_metaclass(ABCMeta)):
         Args:
             filename (str): Specifies the GAC/LAC file to be read.
         """
-        self.filename = os.path.basename(filename)
+        raise NotImplementedError
+
+    @contextmanager
+    def _open(self, filename):
+        """Open the GAC/LAC data file and yield the filehandle.
+
+        Args:
+            filename (str): Path to GAC/LAC file to open
+
+        Note:
+            This method should be called inside the Reader.read implementation
+        """
+        basename = os.path.basename(filename)
+        root, ext = os.path.splitext(basename)
+        if ext == ".gz":
+            self.filename = root
+            file_object = gzip.open(filename, mode='rb')
+        else:
+            self.filename = basename
+            file_object = open(filename, mode='rb')
         LOG.info('Reading %s', self.filename)
+        try:
+            yield file_object
+        finally:
+            file_object.close()
 
     @abstractmethod
     def get_header_timestamp(self):
@@ -385,6 +411,7 @@ class Reader(six.with_metaclass(ABCMeta)):
         """
         if self.lons is None and self.lats is None:
             self.lons, self.lats = self._get_lonlat()
+            self.update_meta_data()
 
             # Interpolate from every eighth pixel to all pixels.
             if self.interpolate_coords:

pygac/tests/test_reader.py

@@ -4,6 +4,7 @@
 # Author(s):
 
 #   Stephan Finkensieper <stephan.finkensieper@dwd.de>
+#   Carlos Horn <carlos.horn@external.eumetsat.int>
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -43,6 +44,17 @@ class TestGacReader(unittest.TestCase):
         self.interpolator = interpolator
         self.reader = GACReader()
 
+    @mock.patch('pygac.reader.open', mock.mock_open(read_data='normal'))
+    @mock.patch('pygac.reader.gzip.open', mock.mock_open(read_data='gzip'))
+    def test__open(self):
+        """Test if proper opener is used."""
+        with self.reader._open("test_file") as f:
+            content_normal = f.read()
+        self.assertEqual(content_normal, 'normal')
+        with self.reader._open("test_file.gz") as f:
+            content_gzip = f.read()
+        self.assertEqual(content_gzip, 'gzip')
+
     def test_to_datetime64(self):
         """Test conversion from (year, jday, msec) to datetime64."""
         t0 = GACReader.to_datetime64(year=np.array(1970), jday=np.array(1),
@@ -61,11 +73,13 @@ class TestGacReader(unittest.TestCase):
         """Test scanline timestamp estimation."""
         self.assertEqual(self.reader.lineno2msec(12345), 6172000)
 
+    @mock.patch('pygac.reader.Reader.update_meta_data')
     @mock.patch('pygac.gac_reader.GACReader._get_lonlat')
     @mock.patch('pygac.gac_reader.GACReader._get_corrupt_mask')
     @mock.patch('pygac.gac_reader.GACReader._adjust_clock_drift')
     def test_get_lonlat(self, adjust_clockdrift,
-                        get_corrupt_mask, get_lonlat):
+                        get_corrupt_mask, get_lonlat,
+                        update_meta_data):
         """Test common lon/lat computation."""
         lon_i = np.array([np.nan, 1, 2, 3, -180.1, 180.1])
         lat_i = np.array([1, 2, 3, np.nan, -90.1, 90.1])
@@ -80,6 +94,7 @@ class TestGacReader(unittest.TestCase):
         # Default
         lons, lats = self.reader.get_lonlat()
         get_lonlat.assert_called()
+        update_meta_data.assert_called()
         adjust_clockdrift.assert_called()
         numpy.testing.assert_array_equal(lons, lons_exp)
         numpy.testing.assert_array_equal(lats, lats_exp)
@@ -117,11 +132,12 @@ class TestGacReader(unittest.TestCase):
         for method in methods:
             method.asser_not_called()
 
+    @mock.patch('pygac.reader.Reader.update_meta_data')
     @mock.patch('pygac.gac_reader.GACReader._get_corrupt_mask')
     @mock.patch('pygac.gac_reader.GACReader._adjust_clock_drift')
     @mock.patch('pygac.gac_reader.GACReader._get_lonlat')
     def test_interpolate(self, _get_lonlat, _adjust_clock_drift,
-                         _get_corrupt_mask):
+                         _get_corrupt_mask, update_meta_data):
         """Test interpolate method in get_lonlat."""
         self.lons = None
         self.lats = None

pygac/version.py

@@ -22,4 +22,4 @@
 
 """Just holding the version of the pygac module."""
 
-__version__ = '1.3.0'
+__version__ = '1.3.1'

setup.cfg

+[flake8]
+max-line-length = 120