Skip to content
Commits on Source (5)
language: python
python:
- '2.7'
- '3.6'
env:
global:
# Set defaults to avoid repeating in most cases
......@@ -15,8 +12,19 @@ env:
- CONDA_CHANNELS='conda-forge'
- CONDA_CHANNEL_PRIORITY=True
matrix:
include:
- env:
- PYTHON_VERSION=2.7
- NUMPY_VERSION=1.16
os: linux
- env: PYTHON_VERSION=3.6
os: linux
- env: PYTHON_VERSION=3.7
os: linux
install:
- git clone --depth 1 -b all-the-fixes git://github.com/djhoese/ci-helpers.git
- git clone --depth 1 git://github.com/astropy/ci-helpers.git
- source ci-helpers/travis/setup_conda.sh
script: coverage run --source=trollimage setup.py test
after_success:
......
## Version 1.11.0 (2019/10/24)
### Pull Requests Merged
#### Bugs fixed
* [PR 58](https://github.com/pytroll/trollimage/pull/58) - Make tags containing values to compute use store for saving
#### Features added
* [PR 60](https://github.com/pytroll/trollimage/pull/60) - Add tests on py 3.7
* [PR 59](https://github.com/pytroll/trollimage/pull/59) - Add scale and offset inclusion utility when rio saving
* [PR 57](https://github.com/pytroll/trollimage/pull/57) - Add the `apply_pil` method
In this release 4 pull requests were closed.
## Version 1.10.1 (2019/09/26)
### Pull Requests Merged
......
......@@ -18,9 +18,13 @@ environment:
PYTHON_ARCH: "64"
NUMPY_VERSION: "stable"
- PYTHON: "C:\\Python37_64"
PYTHON_VERSION: "3.7"
PYTHON_ARCH: "64"
NUMPY_VERSION: "stable"
install:
# - "git clone --depth 1 git://github.com/astropy/ci-helpers.git"
- "git clone --depth 1 -b all-the-fixes git://github.com/djhoese/ci-helpers.git"
- "git clone --depth 1 git://github.com/astropy/ci-helpers.git"
- "powershell ci-helpers/appveyor/install-miniconda.ps1"
- "conda activate test"
......
trollimage (1.10.1-2) UNRELEASED; urgency=medium
trollimage (1.11.0-1) unstable; urgency=medium
* New upstream release.
* Bump Standards-Version to 4.4.1, no changes.
* debian/patches:
- refresh all patches
-- Antonio Valentino <antonio.valentino@tiscali.it> Mon, 30 Sep 2019 20:05:04 +0200
-- Antonio Valentino <antonio.valentino@tiscali.it> Mon, 28 Oct 2019 07:06:03 +0000
trollimage (1.10.1-1) unstable; urgency=medium
......
......@@ -8,10 +8,10 @@ Skip tests that require display.
1 file changed, 1 insertion(+)
diff --git a/trollimage/tests/test_image.py b/trollimage/tests/test_image.py
index 06f5fd0..890bd85 100644
index 83538e5..2b6cbe6 100644
--- a/trollimage/tests/test_image.py
+++ b/trollimage/tests/test_image.py
@@ -1863,6 +1863,7 @@ class TestXRImage(unittest.TestCase):
@@ -1916,6 +1916,7 @@ class TestXRImage(unittest.TestCase):
"""Test putalpha."""
pass
......
......@@ -23,12 +23,15 @@
# along with mpop. If not, see <http://www.gnu.org/licenses/>.
"""Module for testing the image and xrimage modules."""
import os
import sys
import random
import unittest
import sys
import tempfile
import unittest
from collections import OrderedDict
from tempfile import NamedTemporaryFile
import numpy as np
from trollimage import image
try:
......@@ -958,7 +961,7 @@ class TestXRImage(unittest.TestCase):
delay = img.save(tmp.name, compute=False)
self.assertIsInstance(delay, tuple)
self.assertIsInstance(delay[0], da.Array)
self.assertIsInstance(delay[1], xrimage.RIOFile)
self.assertIsInstance(delay[1], xrimage.RIODataset)
da.store(*delay)
delay[1].close()
......@@ -1031,7 +1034,7 @@ class TestXRImage(unittest.TestCase):
delay = img.save(tmp.name, compute=False)
self.assertIsInstance(delay, tuple)
self.assertIsInstance(delay[0], da.Array)
self.assertIsInstance(delay[1], xrimage.RIOFile)
self.assertIsInstance(delay[1], xrimage.RIODataset)
da.store(*delay)
delay[1].close()
......@@ -1198,6 +1201,44 @@ class TestXRImage(unittest.TestCase):
with rio.open(tmp.name) as f:
self.assertEqual(len(f.overviews(1)), 2)
@unittest.skipIf(sys.platform.startswith('win'), "'NamedTemporaryFile' not supported on Windows")
def test_save_tags(self):
"""Test saving geotiffs with tags."""
import xarray as xr
from trollimage import xrimage
import rasterio as rio
# numpy array image
data = xr.DataArray(np.arange(75).reshape(5, 5, 3), dims=[
'y', 'x', 'bands'], coords={'bands': ['R', 'G', 'B']})
img = xrimage.XRImage(data)
tags = {'avg': img.data.mean(), 'current_song': 'disco inferno'}
self.assertTrue(np.issubdtype(img.data.dtype, np.integer))
with NamedTemporaryFile(suffix='.tif') as tmp:
img.save(tmp.name, tags=tags)
tags['avg'] = '37.0'
with rio.open(tmp.name) as f:
self.assertEqual(f.tags(), tags)
@unittest.skipIf(sys.platform.startswith('win'), "'NamedTemporaryFile' not supported on Windows")
def test_save_scale_offset(self):
"""Test saving geotiffs with tags."""
import xarray as xr
from trollimage import xrimage
import rasterio as rio
data = xr.DataArray(np.arange(25).reshape(5, 5, 1), dims=[
'y', 'x', 'bands'], coords={'bands': ['L']})
img = xrimage.XRImage(data)
img.stretch()
with NamedTemporaryFile(suffix='.tif') as tmp:
img.save(tmp.name, include_scale_offset_tags=True)
tags = {'scale': 24.0 / 255, 'offset': 0}
with rio.open(tmp.name) as f:
ftags = f.tags()
for key, val in tags.items():
self.assertAlmostEqual(float(ftags[key]), val)
def test_gamma(self):
"""Test gamma correction."""
import xarray as xr
......@@ -1612,6 +1653,18 @@ class TestXRImage(unittest.TestCase):
self.assertTrue(img2.mode == 'RGBA')
self.assertTrue(len(img2.data.coords['bands']) == 4)
def test_final_mode(self):
"""Test final_mode."""
import xarray as xr
from trollimage import xrimage
# numpy array image
data = xr.DataArray(np.arange(75).reshape(5, 5, 3), dims=[
'y', 'x', 'bands'], coords={'bands': ['R', 'G', 'B']})
img = xrimage.XRImage(data)
self.assertEqual(img.final_mode(None), 'RGBA')
self.assertEqual(img.final_mode(0), 'RGB')
def test_colorize(self):
"""Test colorize with an RGB colormap."""
import xarray as xr
......@@ -1875,6 +1928,42 @@ class TestXRImage(unittest.TestCase):
img.show()
s.assert_called_once()
def test_apply_pil(self):
"""Test the apply_pil method."""
import xarray as xr
from trollimage import xrimage
np_data = np.arange(75).reshape(5, 5, 3) / 75.
data = xr.DataArray(np_data, dims=[
'y', 'x', 'bands'], coords={'bands': ['R', 'G', 'B']})
dummy_args = [(OrderedDict(), ), {}]
def dummy_fun(pil_obj, *args, **kwargs):
dummy_args[0] = args
dummy_args[1] = kwargs
return pil_obj
img = xrimage.XRImage(data)
pi = mock.MagicMock()
img.pil_image = pi
res = img.apply_pil(dummy_fun, 'RGB')
# check that the pil image generation is delayed
pi.assert_not_called()
# make it happen
res.data.data.compute()
pi.return_value.convert.assert_called_with('RGB')
img = xrimage.XRImage(data)
pi = mock.MagicMock()
img.pil_image = pi
res = img.apply_pil(dummy_fun, 'RGB',
fun_args=('Hey', 'Jude'),
fun_kwargs={'chorus': "La lala lalalala"})
self.assertEqual(dummy_args, [({}, ), {}])
res.data.data.compute()
self.assertEqual(dummy_args, [(OrderedDict(), 'Hey', 'Jude'), {'chorus': "La lala lalalala"}])
def suite():
"""Create the suite for test_image."""
......
......@@ -23,9 +23,9 @@ def get_keywords():
# setup.py/versioneer.py will grep for the variable names, so they must
# each be defined on a line of their own. _version.py will just call
# get_keywords().
git_refnames = " (HEAD -> master, tag: v1.10.1)"
git_full = "9130e96fae8e880ccf843298b508712a4d80a481"
git_date = "2019-09-26 15:08:59 -0500"
git_refnames = " (HEAD -> master, tag: v1.11.0)"
git_full = "103b269144d33bbcf106c8afa20de95618d5a890"
git_date = "2019-10-24 20:12:03 +0200"
keywords = {"refnames": git_refnames, "full": git_full, "date": git_date}
return keywords
......
......@@ -34,13 +34,14 @@ chunks can be saved in parallel.
import logging
import os
import threading
import numpy as np
from PIL import Image as PILImage
import xarray as xr
import xarray.ufuncs as xu
import dask
import dask.array as da
from dask.delayed import delayed
from trollimage.image import check_image_format
......@@ -71,47 +72,24 @@ class RIOFile(object):
self.mode = mode
self.kwargs = kwargs
self.rfile = None
self._closed = True
self.overviews = kwargs.pop('overviews', None)
self.lock = threading.Lock()
def __setitem__(self, key, item):
"""Put the data chunk in the image."""
if len(key) == 3:
indexes = list(range(
key[0].start + 1,
key[0].stop + 1,
key[0].step or 1
))
y = key[1]
x = key[2]
else:
indexes = 1
y = key[0]
x = key[1]
chy_off = y.start
chy = y.stop - y.start
chx_off = x.start
chx = x.stop - x.start
# band indexes
self.rfile.write(item, window=Window(chx_off, chy_off, chx, chy),
indexes=indexes)
@property
def closed(self):
"""Check if the file is closed."""
return self.rfile is None or self.rfile.closed
def open(self, mode=None):
"""Open the file."""
mode = mode or self.mode
if self._closed:
if self.closed:
self.rfile = rasterio.open(self.path, mode, **self.kwargs)
self._closed = False
def close(self):
"""Close the file."""
if not self._closed:
if self.overviews:
logger.debug('Building overviews %s', str(self.overviews))
self.rfile.build_overviews(self.overviews)
with self.lock:
if not self.closed:
self.rfile.close()
self._closed = True
def __enter__(self):
"""Enter method."""
......@@ -142,6 +120,82 @@ class RIOFile(object):
else:
self.rfile.colorinterp = val
def write(self, *args, **kwargs):
"""Write to the file."""
with self.lock:
self.open('a')
return self.rfile.write(*args, **kwargs)
def build_overviews(self, *args, **kwargs):
"""Write overviews."""
with self.lock:
self.open('a')
return self.rfile.build_overviews(*args, **kwargs)
def update_tags(self, *args, **kwargs):
"""Update tags."""
with self.lock:
self.open('a')
return self.rfile.update_tags(*args, **kwargs)
class RIOTag:
"""Rasterio wrapper to allow da.store on tag."""
def __init__(self, rfile, name):
"""Init the rasterio tag."""
self.rfile = rfile
self.name = name
def __setitem__(self, key, item):
"""Put the data in the tag."""
kwargs = {self.name: item.item()}
self.rfile.update_tags(**kwargs)
def close(self):
"""Close the file."""
return self.rfile.close()
class RIODataset:
"""A wrapper for a rasterio dataset."""
def __init__(self, rfile, overviews=None):
"""Init the rasterio dataset."""
self.rfile = rfile
self.overviews = overviews
def __setitem__(self, key, item):
"""Put the data chunk in the image."""
if len(key) == 3:
indexes = list(range(
key[0].start + 1,
key[0].stop + 1,
key[0].step or 1
))
y = key[1]
x = key[2]
else:
indexes = 1
y = key[0]
x = key[1]
chy_off = y.start
chy = y.stop - y.start
chx_off = x.start
chx = x.stop - x.start
# band indexes
self.rfile.write(item, window=Window(chx_off, chy_off, chx, chy),
indexes=indexes)
def close(self):
"""Close the file."""
if self.overviews:
logger.debug('Building overviews %s', str(self.overviews))
self.rfile.build_overviews(self.overviews)
return self.rfile.close()
def color_interp(data):
"""Get the color interpretation for this image."""
......@@ -170,6 +224,45 @@ def color_interp(data):
return [colors[band] for band in data['bands'].values]
def combine_scales_offsets(*args):
"""Combine ``(scale, offset)`` tuples in one, considering they are applied from left to right.
For example, if we have our base data called ```orig_data`` and apply to it
``(scale_1, offset_1)``, then ``(scale_2, offset_2)`` such that::
data_1 = orig_data * scale_1 + offset_1
data_2 = data_1 * scale_2 + offset_2
this function will return the tuple ``(scale, offset)`` such that::
data_2 = orig_data * scale + offset
given the arguments ``(scale_1, offset_1), (scale_2, offset_2)``.
"""
cscale = 1
coffset = 0
for scale, offset in args:
cscale *= scale
coffset = coffset * scale + offset
return cscale, coffset
def invert_scale_offset(scale, offset):
"""Invert scale and offset to allow reverse transformation.
Ie, it will return ``rscale, roffset`` such that::
orig_data = rscale * data + roffset
if::
data = scale * orig_data + offset
"""
return 1 / scale, -offset / scale
class XRImage(object):
"""Image class using an :class:`xarray.DataArray` as internal storage.
......@@ -287,14 +380,37 @@ class XRImage(object):
def rio_save(self, filename, fformat=None, fill_value=None,
dtype=np.uint8, compute=True, tags=None,
keep_palette=False, cmap=None,
keep_palette=False, cmap=None, overviews=None,
include_scale_offset_tags=False,
**format_kwargs):
"""Save the image using rasterio.
Overviews can be added to the file using the `overviews` kwarg, eg::
Args:
filename (string): The filename to save to.
fformat (string): The format to save to. If not specified (default),
it will be infered from the file extension.
fill_value (number): The value to fill the missing data with.
Default is ``None``, translating to trying to keep the data
transparent.
dtype (np.dtype): The type to save the data to. Defaults to
np.uint8.
compute (bool): Whether (default) or not to compute the lazy data.
tags (dict): Tags to include in the file.
keep_palette (bool): Whether or not (default) to keep the image in
P mode.
cmap (colormap): The colormap to use for the data.
overviews (list): The reduction factors of the overviews to include
in the image, eg::
img.rio_save('myfile.tif', overviews=[2, 4, 8, 16])
include_scale_offset_tags (bool): Whether or not (default) to
include a ``scale`` and an ``offset`` tag in the data that would
help retrieving original data values from pixel values.
Returns:
The delayed or computed result of the saving.
"""
fformat = fformat or os.path.splitext(filename)[1][1:]
drivers = {'jpg': 'JPEG',
......@@ -309,7 +425,6 @@ class XRImage(object):
data, mode = self.finalize(fill_value, dtype=dtype,
keep_palette=keep_palette, cmap=cmap)
data = data.transpose('bands', 'y', 'x')
data.attrs = self.data.attrs
crs = None
gcps = None
......@@ -361,6 +476,10 @@ class XRImage(object):
elif driver == 'JPEG' and 'A' in mode:
raise ValueError('JPEG does not support alpha')
if include_scale_offset_tags:
scale, offset = self.get_scaling_from_history(data.attrs.get('enhancement_history', []))
tags['scale'], tags['offset'] = invert_scale_offset(scale, offset)
# FIXME add metadata
r_file = RIOFile(filename, 'w', driver=driver,
width=data.sizes['x'], height=data.sizes['y'],
......@@ -374,7 +493,6 @@ class XRImage(object):
r_file.open()
if not keep_palette:
r_file.colorinterp = color_interp(data)
r_file.rfile.update_tags(**tags)
if keep_palette and cmap is not None:
if data.dtype != 'uint8':
......@@ -386,15 +504,35 @@ class XRImage(object):
except AttributeError:
raise ValueError("Colormap is not formatted correctly")
da_tags = []
for key, val in list(tags.items()):
try:
if isinstance(val.data, da.Array):
da_tags.append((val.data, RIOTag(r_file, key)))
tags.pop(key)
except AttributeError:
continue
r_file.rfile.update_tags(**tags)
r_dataset = RIODataset(r_file, overviews)
to_store = (data.data, r_dataset)
if da_tags:
to_store = list(zip(*([to_store] + da_tags)))
if compute:
# write data to the file now
res = da.store(data.data, r_file)
r_file.close()
res = da.store(*to_store)
to_close = to_store[1]
if not isinstance(to_close, tuple):
to_close = [to_close]
for item in to_close:
item.close()
return res
# provide the data object and the opened file so the caller can
# store them when they would like. Caller is responsible for
# closing the file
return data.data, r_file
return to_store
def pil_save(self, filename, fformat=None, fill_value=None,
compute=True, **format_kwargs):
......@@ -417,6 +555,62 @@ class XRImage(object):
return delay.compute()
return delay
def get_scaling_from_history(self, history=None):
"""Merge the scales and offsets from the history.
If ``history`` isn't provided, the history of the current image will be
used.
"""
if history is None:
history = self.data.attrs.get('enhancement_history', [])
try:
scaling = [(item['scale'], item['offset']) for item in history]
except KeyError as err:
raise NotImplementedError('Can only get combine scaling from a list of scaling operations: %s' % str(err))
return combine_scales_offsets(*scaling)
@delayed(nout=1, pure=True)
def _delayed_apply_pil(self, fun, pil_args, pil_kwargs, fun_args, fun_kwargs,
image_metadata=None, output_mode=None):
if pil_args is None:
pil_args = tuple()
if pil_kwargs is None:
pil_kwargs = dict()
if fun_args is None:
fun_args = tuple()
if fun_kwargs is None:
fun_kwargs = dict()
if image_metadata is None:
image_metadata = dict()
new_img = fun(self.pil_image(*pil_args, **pil_kwargs), image_metadata, *fun_args, **fun_kwargs)
if output_mode is not None:
new_img = new_img.convert(output_mode)
return np.array(new_img) / self.data.dtype.type(255.0)
def apply_pil(self, fun, output_mode, pil_args=None, pil_kwargs=None, fun_args=None, fun_kwargs=None):
"""Apply a function `fun` on the pillow image corresponding to the instance of the XRImage.
The function shall take a pil image as first argument, and is then passed fun_args and fun_kwargs.
In addition, the current images's metadata is passed as a keyword argument called `image_mda`.
It is expected to return the modified pil image.
This function returns a new XRImage instance with the modified image data.
The pil_args and pil_kwargs are passed to the `pil_image` method of the XRImage instance.
"""
new_array = self._delayed_apply_pil(fun, pil_args, pil_kwargs, fun_args, fun_kwargs,
self.data.attrs, output_mode)
bands = len(output_mode)
arr = da.from_delayed(new_array, dtype=self.data.dtype,
shape=(self.data.sizes['y'], self.data.sizes['x'], bands))
new_data = xr.DataArray(arr, dims=['y', 'x', 'bands'],
coords={'y': self.data.coords['y'],
'x': self.data.coords['x'],
'bands': list(output_mode)},
attrs=self.data.attrs)
return XRImage(new_data)
def _pngmeta(self):
"""Return GeoImage.tags as a PNG metadata object.
......@@ -485,7 +679,9 @@ class XRImage(object):
if np.issubdtype(data.dtype, np.integer):
# xarray sometimes upcasts this calculation, so cast again
null_mask = self._scale_to_dtype(null_mask, data.dtype).astype(data.dtype)
attrs = data.attrs.copy()
data = xr.concat([data, null_mask], dim="bands")
data.attrs = attrs
return data
def _scale_to_dtype(self, data, dtype):
......@@ -497,6 +693,7 @@ class XRImage(object):
be in the 0-1 range already.
"""
attrs = data.attrs.copy()
if np.issubdtype(dtype, np.integer):
if np.issubdtype(data, np.integer):
# preserve integer data type
......@@ -504,8 +701,12 @@ class XRImage(object):
else:
# scale float data (assumed to be 0 to 1) to full integer space
dinfo = np.iinfo(dtype)
data = data.clip(0, 1) * (dinfo.max - dinfo.min) + dinfo.min
scale = dinfo.max - dinfo.min
offset = dinfo.min
data = data.clip(0, 1) * scale + offset
attrs.setdefault('enhancement_history', list()).append({'scale': scale, 'offset': offset})
data = data.round()
data.attrs = attrs
return data
def _check_modes(self, modes):
......@@ -616,6 +817,13 @@ class XRImage(object):
DeprecationWarning)
return self.finalize(fill_value, dtype, keep_palette, cmap)
def final_mode(self, fill_value=None):
"""Get the mode of the finalized image when provided this fill_value."""
if fill_value is None and not self.mode.endswith('A'):
return self.mode + 'A'
else:
return self.mode
def finalize(self, fill_value=None, dtype=np.uint8, keep_palette=False, cmap=None):
"""Finalize the image to be written to an output file.
......@@ -645,19 +853,29 @@ class XRImage(object):
"setting fill_value to 0")
fill_value = 0
final_data = self.data
final_data = self.data.copy()
try:
final_data.attrs['enhancement_history'] = list(self.data.attrs['enhancement_history'])
except KeyError:
pass
attrs = final_data.attrs
# if the data are integers then this fill value will be used to check for invalid values
with xr.set_options(keep_attrs=True):
ifill = final_data.attrs.get('_FillValue') if np.issubdtype(final_data, np.integer) else None
if not keep_palette:
if fill_value is None and not self.mode.endswith('A'):
# We don't have a fill value or an alpha, let's add an alpha
alpha = self._create_alpha(final_data, fill_value=ifill)
final_data = self._scale_to_dtype(final_data, dtype).astype(dtype)
final_data = self._scale_to_dtype(final_data, dtype)
attrs = final_data.attrs
final_data = final_data.astype(dtype)
final_data = self._add_alpha(final_data, alpha=alpha)
final_data.attrs = attrs
else:
# scale float data to the proper dtype
# this method doesn't cast yet so that we can keep track of NULL values
final_data = self._scale_to_dtype(final_data, dtype)
attrs = final_data.attrs
# Add fill_value after all other calculations have been done to
# make sure it is not scaled for the data type
if ifill is not None and fill_value is not None:
......@@ -669,7 +887,8 @@ class XRImage(object):
final_data = final_data.fillna(dtype(fill_value))
final_data = final_data.astype(dtype)
final_data.attrs = self.data.attrs
final_data.attrs = attrs
return final_data, ''.join(final_data['bands'].values)
def pil_image(self, fill_value=None, compute=True):
......@@ -929,7 +1148,7 @@ class XRImage(object):
"""
attrs = self.data.attrs
self.data = k * xu.log(self.data / s0)
self.data = k * np.log(self.data / s0)
self.data.attrs = attrs
self.data.attrs.setdefault('enhancement_history', []).append({'weber_fechner': (k, s0)})
......