Commit 921e36b8 authored by Ole Streicher's avatar Ole Streicher

New upstream version 1.2.0

parent a9503fd3
1.1.0 (2016-03-03)
1.2.0(2016-10-04)
-----------------
- Added Tabular model. [#214]
- Forced new blocks to be contiguous [#221]
- Rewrote code which tags complex objects [#223]
- Fixed version error message [#224]
1.0.5 (2016-06-28)
------------------
- Renamed repository to asdf.
- Fixed a memory leak when reading wcs that grew memory to over 10 Gb. [#200]
1.0.4 (2016-05-25)
------------------
- Added wrapper class for astropy.core.Time, TaggedTime. [#198]
1.0.2 (2016-02-29)
------------------
......
Metadata-Version: 1.0
Name: asdf
Version: 1.1.0
Version: 1.2.0
Summary: Python tools to handle ASDF files
Home-page: http://github.com/spacetelescope/asdf
Author: Erik Bray, Michael Droettboom
......
asdf
====
.. image:: https://readthedocs.org/projects/pyasdf/badge/?version=latest
:target: http://pyasdf.readthedocs.io/en/latest/?badge=latest
:alt: Documentation Status
Python library for reading and writing ASDF files.
.. image:: https://coveralls.io/repos/spacetelescope/asdf/badge.svg?branch=master&service=github
:target: https://coveralls.io/github/spacetelescope/asdf?branch=master
.. image:: https://travis-ci.org/spacetelescope/asdf.svg?branch=master
:target: https://travis-ci.org/spacetelescope/asdf
.. image:: http://img.shields.io/badge/powered%20by-AstroPy-orange.svg?style=flat
:target: http://www.astropy.org
:alt: Powered by Astropy Badge
Python library for reading and writing ASDF files.
Advanced Scientific Data Format (ASDF) is a next generation
interchange format for scientific data.
Installation
------------
To clone the project from github and initialize the asdf-standard submodule::
git clone https://github.com/spacetelescope/asdf.git
cd asdf/asdf_standard
git submodule init
git submodule update
To install::
python setup.py install
Testing
-------
To run the unit tests::
python setup.py test
Coverage
--------
.. image:: https://coveralls.io/repos/spacetelescope/asdf/badge.svg?branch=master&service=github
:target: https://coveralls.io/github/spacetelescope/asdf?branch=master
......@@ -409,7 +409,7 @@ class _Bootstrapper(object):
def get_index_dist(self):
if not self.download:
log.warn('Downloading {0!r} disabled.'.format(DIST_NAME))
return False
return None
log.warn(
"Downloading {0!r}; run setup.py with the --offline option to "
......
......@@ -3,8 +3,8 @@
%YAML 1.1
%TAG ! tag:stsci.edu:asdf/
--- !core/asdf-1.0.0
asdf_library: !core/software-1.0.0 {author: Space Telescope Science Institute, homepage: 'http://github.com/spacetelescope/pyasdf',
name: pyasdf, version: 1.0.0}
asdf_library: !core/software-1.0.0 {author: Space Telescope Science Institute, homepage: 'http://github.com/spacetelescope/asdf',
name: asdf, version: 1.0.0}
data: !core/ndarray-1.0.0
data: ['', ascii]
datatype: [ascii, 5]
......
......@@ -3,8 +3,8 @@
%YAML 1.1
%TAG ! tag:stsci.edu:asdf/
--- !core/asdf-1.0.0
asdf_library: !core/software-1.0.0 {author: Space Telescope Science Institute, homepage: 'http://github.com/spacetelescope/pyasdf',
name: pyasdf, version: 1.0.0}
asdf_library: !core/software-1.0.0 {author: Space Telescope Science Institute, homepage: 'http://github.com/spacetelescope/asdf',
name: asdf, version: 1.0.0}
data: !core/ndarray-1.0.0
data: [0, 1, 2, 3, 4, 5, 6, 7]
datatype: int64
......
......@@ -3,8 +3,8 @@
%YAML 1.1
%TAG ! tag:stsci.edu:asdf/
--- !core/asdf-1.0.0
asdf_library: !core/software-1.0.0 {author: Space Telescope Science Institute, homepage: 'http://github.com/spacetelescope/pyasdf',
name: pyasdf, version: 1.0.0}
asdf_library: !core/software-1.0.0 {author: Space Telescope Science Institute, homepage: 'http://github.com/spacetelescope/asdf',
name: asdf, version: 1.0.0}
datatype<c16: !core/ndarray-1.0.0
data: [!core/complex-1.0.0 0j, !core/complex-1.0.0 0j, !core/complex-1.0.0 (nan+nanj),
!core/complex-1.0.0 (nan+infj), !core/complex-1.0.0 (nan-infj), !core/complex-1.0.0 -1.79769313486e+308j,
......
......@@ -3,8 +3,8 @@
%YAML 1.1
%TAG ! tag:stsci.edu:asdf/
--- !core/asdf-1.0.0
asdf_library: !core/software-1.0.0 {author: Space Telescope Science Institute, homepage: 'http://github.com/spacetelescope/pyasdf',
name: pyasdf, version: 1.0.0}
asdf_library: !core/software-1.0.0 {author: Space Telescope Science Institute, homepage: 'http://github.com/spacetelescope/asdf',
name: asdf, version: 1.0.0}
bzp2: !core/ndarray-1.0.0
data: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40,
......
......@@ -3,8 +3,8 @@
%YAML 1.1
%TAG ! tag:stsci.edu:asdf/
--- !core/asdf-1.0.0
asdf_library: !core/software-1.0.0 {author: Space Telescope Science Institute, homepage: 'http://github.com/spacetelescope/pyasdf',
name: pyasdf, version: 1.0.0}
asdf_library: !core/software-1.0.0 {author: Space Telescope Science Institute, homepage: 'http://github.com/spacetelescope/asdf',
name: asdf, version: 1.0.0}
data: !core/ndarray-1.0.0
source: exploded0000.asdf
datatype: int64
......
......@@ -3,8 +3,8 @@
%YAML 1.1
%TAG ! tag:stsci.edu:asdf/
--- !core/asdf-1.0.0
asdf_library: !core/software-1.0.0 {author: Space Telescope Science Institute, homepage: 'http://github.com/spacetelescope/pyasdf',
name: pyasdf, version: 1.0.0}
asdf_library: !core/software-1.0.0 {author: Space Telescope Science Institute, homepage: 'http://github.com/spacetelescope/asdf',
name: asdf, version: 1.0.0}
data: !core/ndarray-1.0.0
data: [0, 1, 2, 3, 4, 5, 6, 7]
datatype: int64
......
......@@ -3,8 +3,8 @@
%YAML 1.1
%TAG ! tag:stsci.edu:asdf/
--- !core/asdf-1.0.0
asdf_library: !core/software-1.0.0 {author: Space Telescope Science Institute, homepage: 'http://github.com/spacetelescope/pyasdf',
name: pyasdf, version: 1.0.0}
asdf_library: !core/software-1.0.0 {author: Space Telescope Science Institute, homepage: 'http://github.com/spacetelescope/asdf',
name: asdf, version: 1.0.0}
datatype<f4: !core/ndarray-1.0.0
data: [0.0, -0.0, .nan, .inf, -.inf, -3.4028234663852886e+38, 3.4028234663852886e+38,
1.1920928955078125e-07, 5.960464477539063e-08, 1.1754943508222875e-38]
......
......@@ -3,8 +3,8 @@
%YAML 1.1
%TAG ! tag:stsci.edu:asdf/
--- !core/asdf-1.0.0
asdf_library: !core/software-1.0.0 {author: Space Telescope Science Institute, homepage: 'http://github.com/spacetelescope/pyasdf',
name: pyasdf, version: 1.0.0}
asdf_library: !core/software-1.0.0 {author: Space Telescope Science Institute, homepage: 'http://github.com/spacetelescope/asdf',
name: asdf, version: 1.0.0}
datatype<i1: !core/ndarray-1.0.0
data: [127, -128, 0]
datatype: int8
......
......@@ -3,8 +3,8 @@
%YAML 1.1
%TAG ! tag:stsci.edu:asdf/
--- !core/asdf-1.0.0
asdf_library: !core/software-1.0.0 {author: Space Telescope Science Institute, homepage: 'http://github.com/spacetelescope/pyasdf',
name: pyasdf, version: 1.0.0}
asdf_library: !core/software-1.0.0 {author: Space Telescope Science Institute, homepage: 'http://github.com/spacetelescope/asdf',
name: asdf, version: 1.0.0}
data: !core/ndarray-1.0.0
data: [0, 1, 2, 3, 4, 5, 6, 7]
datatype: int64
......
......@@ -3,8 +3,8 @@
%YAML 1.1
%TAG ! tag:stsci.edu:asdf/
--- !core/asdf-1.0.0
asdf_library: !core/software-1.0.0 {author: Space Telescope Science Institute, homepage: 'http://github.com/spacetelescope/pyasdf',
name: pyasdf, version: 1.0.0}
asdf_library: !core/software-1.0.0 {author: Space Telescope Science Institute, homepage: 'http://github.com/spacetelescope/asdf',
name: asdf, version: 1.0.0}
my_stream: !core/ndarray-1.0.0
data:
- [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
......
......@@ -3,8 +3,8 @@
%YAML 1.1
%TAG ! tag:stsci.edu:asdf/
--- !core/asdf-1.0.0
asdf_library: !core/software-1.0.0 {author: Space Telescope Science Institute, homepage: 'http://github.com/spacetelescope/pyasdf',
name: pyasdf, version: 1.0.0}
asdf_library: !core/software-1.0.0 {author: Space Telescope Science Institute, homepage: 'http://github.com/spacetelescope/asdf',
name: asdf, version: 1.0.0}
datatype<U: !core/ndarray-1.0.0
data: ['', Æʩ]
datatype: [ucs4, 2]
......
......@@ -3,8 +3,8 @@
%YAML 1.1
%TAG ! tag:stsci.edu:asdf/
--- !core/asdf-1.0.0
asdf_library: !core/software-1.0.0 {author: Space Telescope Science Institute, homepage: 'http://github.com/spacetelescope/pyasdf',
name: pyasdf, version: 1.0.0}
asdf_library: !core/software-1.0.0 {author: Space Telescope Science Institute, homepage: 'http://github.com/spacetelescope/asdf',
name: asdf, version: 1.0.0}
datatype<U: !core/ndarray-1.0.0
data: ['', "\U00010020"]
datatype: [ucs4, 2]
......
%YAML 1.1
---
$schema: "http://stsci.edu/schemas/yaml-schema/draft-01"
id: "http://stsci.edu/schemas/asdf/transform/tabular-1.0.0"
tag: "tag:stsci.edu:asdf/transform/tabular-1.0.0"
title: >
A Tabular model.
description: |
Tabular represents a lookup table with values corresponding to
some grid points.
It computes the interpolated values corresponding to the given
inputs. Three methods of interpolation are supported -
"linear", "nearest" and "splinef2d". It supports extrapolation.
type: object
properties:
lookup_table:
description: >
Table values.
anyOf:
- $ref: ../core/ndarray-1.0.0
- type: array
points:
type: array
items:
anyOf:
- type: array
- $ref: ../core/ndarray-1.0.0
description: |
Grid values - each row in the array corresponds to a dimension
in the lookup table. The grid does not have to be regular.
method:
description: |
Method of interpolation. Supported are "linear" and
"nearest", and "splinef2d". "splinef2d" is only supported for
2-dimensional data.
type: string
enum: ["linear", "nearest", "splinef2d"]
default: "linear"
bounds_error:
description: |
If True, when interpolated values are requested outside of the
domain of the input data, a ValueError is raised.
If False, then "fill_value" is used.
type: boolean
default: true
fill_value:
description: |
If provided, the value to use for points outside of the
interpolation domain. If None, values outside the domain are
extrapolated. Extrapolation is not supported by method "splinef2d".
type: number
required: [lookup_table]
\ No newline at end of file
......@@ -14,7 +14,7 @@ Data Format (ASDF) files
from ._internal_init import *
# ----------------------------------------------------------------------------
if _PYASDF_SETUP_ is False:
if _ASDF_SETUP_ is False:
__all__ = ['AsdfFile', 'AsdfType', 'AsdfExtension',
'Stream', 'open', 'test', 'commands',
'ValidationError']
......
......@@ -7,14 +7,14 @@ __all__ = ['__version__', '__githash__', 'test']
# this indicates whether or not we are in the package's setup.py
try:
_PYASDF_SETUP_
_ASDF_SETUP_
except NameError:
from sys import version_info
if version_info[0] >= 3:
import builtins
else:
import __builtin__ as builtins
builtins._PYASDF_SETUP_ = False
builtins._ASDF_SETUP_ = False
try:
from .version import version as __version__
......
......@@ -460,7 +460,7 @@ class AsdfType(object):
`to_tree`, allows types to customize how the result is tagged.
"""
obj = cls.to_tree(node, ctx)
return tagged.tag_object(cls.yaml_tag, obj)
return tagged.tag_object(cls.yaml_tag, obj, ctx=ctx)
@classmethod
def from_tree(cls, tree, ctx):
......
......@@ -757,7 +757,10 @@ class Block(object):
])
def __init__(self, data=None, uri=None, array_storage='internal'):
self._data = data
if isinstance(data, np.ndarray) and not data.flags.c_contiguous:
self._data = np.ascontiguousarray(data)
else:
self._data = data
self._uri = uri
self._array_storage = array_storage
......@@ -791,7 +794,7 @@ class Block(object):
@allocated.setter
def allocated(self, allocated):
self._allocated = allocated
@property
def header_size(self):
return self._header.size + constants.BLOCK_HEADER_BOILERPLATE_SIZE
......
......@@ -9,11 +9,31 @@ from __future__ import absolute_import, division, unicode_literals, print_functi
from astropy.tests.pytest_plugins import *
## Uncomment the following line to treat all DeprecationWarnings as
## exceptions
import multiprocessing
import os
import shutil
import tempfile
import pytest
import six
from .extern.RangeHTTPServer import RangeHTTPRequestHandler
# This is to figure out the affiliated package version, rather than
# using Astropy's
from . import version
packagename = os.path.basename(os.path.dirname(__file__))
TESTED_VERSIONS[packagename] = version.version
# Uncomment the following line to treat all DeprecationWarnings as
# exceptions
enable_deprecations_as_exceptions()
try:
PYTEST_HEADER_MODULES['Astropy'] = 'astropy'
PYTEST_HEADER_MODULES['jsonschema'] = 'jsonschema'
PYTEST_HEADER_MODULES['pyyaml'] = 'yaml'
PYTEST_HEADER_MODULES['six'] = 'six'
......@@ -24,16 +44,6 @@ except (NameError, KeyError):
pass
import multiprocessing
import os
import shutil
import tempfile
import pytest
import six
from .extern.RangeHTTPServer import RangeHTTPRequestHandler
def run_server(queue, tmpdir, handler_class): # pragma: no cover
......
......@@ -800,12 +800,11 @@ class MemoryIO(RandomAccessFile):
fd.seek(tell, 0)
def read_into_array(self, size):
result = np.frombuffer(
self._fd.getvalue(), np.uint8, size, self._fd.tell())
# When creating an array from a buffer, it is read-only.
# If we need a read/write array, we have to copy it.
if 'w' in self._mode:
result = result.copy()
buf = self._fd.getvalue()
offset = self._fd.tell()
result = np.frombuffer(buf, np.uint8, size, offset)
# Copy the buffer so the original memory can be released.
result = result.copy()
self.seek(size, SEEK_CUR)
return result
......
......@@ -37,7 +37,6 @@ import six
from .compat import UserDict, UserList, UserString
__all__ = ['tag_object', 'get_tag']
......@@ -98,24 +97,29 @@ class TaggedString(Tagged, UserString, six.text_type):
self._tag == other._tag)
def tag_object(tag, instance):
def tag_object(tag, instance, ctx=None):
"""
Tag an object by wrapping it in a ``Tagged`` instance.
"""
if isinstance(instance, Tagged):
instance._tag = tag
return instance
elif isinstance(instance, dict):
return TaggedDict(instance, tag)
instance = TaggedDict(instance, tag)
elif isinstance(instance, list):
return TaggedList(instance, tag)
instance = TaggedList(instance, tag)
elif isinstance(instance, six.string_types):
instance = TaggedString(instance)
instance._tag = tag
return instance
else:
raise TypeError(
"Don't know how to tag a {0}".format(type(instance)))
from . import AsdfFile, yamlutil
if ctx is None:
ctx = AsdfFile()
try:
instance = yamlutil.custom_tree_to_tagged_tree(instance, ctx)
except TypeError:
raise TypeError("Don't know how to tag a {0}".format(type(instance)))
instance._tag = tag
return instance
def get_tag(instance):
......
......@@ -7,3 +7,4 @@ from .basic import *
from .compound import *
from .projections import *
from .polynomial import *
from .tabular import *
......@@ -51,9 +51,9 @@ class TransformType(AsdfType):
@classmethod
def _to_tree_base_transform_members(cls, model, node, ctx):
if getattr(model, '_custom_inverse', None) is not None:
if getattr(model, '_user_inverse', None) is not None:
node['inverse'] = yamlutil.custom_tree_to_tagged_tree(
model._custom_inverse, ctx)
model._user_inverse, ctx)
if model.name is not None:
node['name'] = model.name
......@@ -62,7 +62,8 @@ class TransformType(AsdfType):
# domains, get this from somewhere else.
domain = model.meta.get('domain')
if domain:
domain = [tagged.tag_object(DomainType.yaml_tag, x) for x in domain]
domain = [tagged.tag_object(DomainType.yaml_tag, x, ctx=ctx)
for x in domain]
node['domain'] = domain
@classmethod
......
......@@ -87,7 +87,7 @@ class CompoundType(TransformType):
except KeyError:
raise ValueError("Unknown operator '{0}'".format(tree.value))
node = tagged.tag_object(cls.make_yaml_tag(tag_name), node)
node = tagged.tag_object(cls.make_yaml_tag(tag_name), node, ctx=ctx)
return node
@classmethod
......
......@@ -25,7 +25,7 @@ class ShiftType(TransformType):
offset = node['offset']
if not np.isscalar(offset):
raise NotImplementedError(
"Pyasdf currently only supports scalar inputs to Shift transform.")
"Asdf currently only supports scalar inputs to Shift transform.")
return modeling.models.Shift(offset)
......@@ -56,7 +56,7 @@ class ScaleType(TransformType):
factor = node['factor']
if not np.isscalar(factor):
raise NotImplementedError(
"Pyasdf currently only supports scalar inputs to Scale transform.")
"Asdf currently only supports scalar inputs to Scale transform.")
return modeling.models.Scale(factor)
......@@ -106,7 +106,7 @@ class PolynomialType(TransformType):
model = modeling.models.Polynomial2D(degree, **coeffs)
else:
raise NotImplementedError(
"Pyasdf currently only supports 1D or 2D polynomial transform.")
"Asdf currently only supports 1D or 2D polynomial transform.")
return model
@classmethod
......
......@@ -102,17 +102,31 @@ class Rotate3DType(TransformType):
from astropy import modeling
if isinstance(model, modeling.rotations.RotateNative2Celestial):
return {"phi": model.lon.value,
"theta": model.lat.value,
"psi": model.lon_pole.value,
"direction": "native2celestial"
}
try:
return {"phi": model.lon.value,
"theta": model.lat.value,
"psi": model.lon_pole.value,
"direction": "native2celestial"
}
except AttributeError:
return {"phi": model.lon,
"theta": model.lat,
"psi": model.lon_pole,
"direction": "native2celestial"
}
elif isinstance(model, modeling.rotations.RotateCelestial2Native):
return {"phi": model.lon.value,
"theta": model.lat.value,
"psi": model.lon_pole.value,
"direction": "celestial2native"
}
try:
return {"phi": model.lon.value,
"theta": model.lat.value,
"psi": model.lon_pole.value,
"direction": "celestial2native"
}
except AttributeError:
return {"phi": model.lon,
"theta": model.lat,
"psi": model.lon_pole,
"direction": "celestial2native"
}
else:
return {"phi": model.phi.value,
"theta": model.theta.value,
......
# Licensed under a 3-clause BSD style license - see LICENSE.rst
# -*- coding: utf-8 -*-
from __future__ import absolute_import, division, unicode_literals, print_function
import numpy as np
from numpy.testing import assert_array_equal
from ... import yamlutil
from .basic import TransformType
__all__ = ['TabularType']
class TabularType(TransformType):
import astropy
name = "transform/tabular"
types = [astropy.modeling.models.Tabular2D,
astropy.modeling.models.Tabular1D
]
@classmethod
def from_tree_transform(cls, node, ctx):
from astropy import modeling
lookup_table = node.pop("lookup_table")
dim = lookup_table.ndim
name = node.get('name', None)
fill_value = node.pop("fill_value", None)
if dim == 1:
# The copy is necessary because the array is memory mapped.
points = (node['points'][0][:],)
model = modeling.models.Tabular1D(points=points, lookup_table=lookup_table,
method=node['method'], bounds_error=node['bounds_error'],
fill_value=fill_value, name=name)
elif dim == 2:
points = tuple([p[:] for p in node['points']])
model = modeling.models.Tabular2D(points=points, lookup_table=lookup_table,
method=node['method'], bounds_error=node['bounds_error'],
fill_value=fill_value, name=name)
else:
tabular_class = modeling.models.tabular_model(dim, name)
points = tuple([p[:] for p in node['points']])
model = tabular_class(points=points, lookup_table=lookup_table,
method=node['method'], bounds_error=node['bounds_error'],
fill_value=fill_value, name=name)
return model
@classmethod
def to_tree_transform(cls, model, ctx):
node = {}
node["fill_value"] = model.fill_value
node["lookup_table"] = model.lookup_table
node["points"] = [p for p in model.points]
node["method"] = str(model.method)
node["bounds_error"] = model.bounds_error
node["name"] = model.name
return yamlutil.custom_tree_to_tagged_tree(node, ctx)
@classmethod
def assert_equal(cls, a, b):
assert_array_equal(a.lookup_table, b.lookup_table)
assert_array_equal(a.points, b.points)
assert (a.method == b.method)
if a.fill_value is None:
assert b.fill_value is None
elif np.isnan(a.fill_value):
assert np.isnan(b.fill_value)
else:
assert(a.fill_value == b.fill_value)
assert(a.bounds_error == b.bounds_error)
......@@ -3,6 +3,8 @@
from __future__ import absolute_import, division, unicode_literals, print_function
import numpy as np
try:
import astropy
except ImportError:
......@@ -126,3 +128,20 @@ def test_generic_projections(tmpdir):
}
helpers.assert_roundtrip_tree(tree, tmpdir)
@pytest.mark.skipif('not HAS_ASTROPY')
def test_tabular_model(tmpdir):
points = np.arange(0, 5)
values = [1., 10, 2, 45, -3]
model = astmodels.Tabular1D(points=points, lookup_table=values)
tree = {'model': model}