Commit dd2ab4d0 authored by Ole Streicher's avatar Ole Streicher

New upstream version 0.1+2017.11.10

parent 2a9f27b0
......@@ -7,6 +7,7 @@ import java.io.EOFException;
import java.io.IOException;
import java.io.UTFDataFormatException;
import java.lang.reflect.Array;
import java.util.logging.Logger;
import nom.tam.util.ArrayDataInput;
import nom.tam.util.ArrayDataOutput;
......@@ -49,6 +50,8 @@ import nom.tam.util.ArrayDataOutput;
public abstract class AbstractArrayDataIO
implements ArrayDataInput, ArrayDataOutput {
private static Logger logger_ = Logger.getLogger( "uk.ac.starlink.fits" );
/**
* Reads one byte from the current position.
*
......@@ -310,11 +313,29 @@ public abstract class AbstractArrayDataIO
}
public int readArray( Object o ) throws IOException {
int nread = 0;
long nread = readLArray( o );
if ( (int) nread == nread ) {
return (int) nread;
}
else {
String msg = new StringBuffer()
.append( "Read too many objects to report (" )
.append( nread )
.append( ">" )
.append( Integer.MAX_VALUE )
.append( ") - should use readLArray instead" )
.toString();
logger_.warning( msg );
return Integer.MAX_VALUE;
}
}
public long readLArray( Object o ) throws IOException {
long nread = 0;
return primitiveArrayRecurse( o, nread );
}
private int primitiveArrayRecurse( Object o, int nread )
private long primitiveArrayRecurse( Object o, long nread )
throws IOException {
if ( o == null ) {
return nread;
......
......@@ -184,7 +184,7 @@ public abstract class AbstractFitsTableWriter extends StreamStarTableWriter
"stil.version", null ),
"Version of STIL software" );
hdr.addValue( "STILCLAS", getClass().getName(),
"Author class in STIL software" );
"STIL Author class" );
}
catch ( HeaderCardException e ) {
logger_.warning( "Trouble adding metadata header cards " + e );
......
package uk.ac.starlink.fits;
import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.EOFException;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
/**
* BasicInput implementation based on a RandomAccessFile.
* Buffering of the underlying file is performed on the assumption that
* access will be largely sequential.
*
* @author Mark Taylor
* @since 8 Nov 2017
*/
public class BufferedRandomInput implements BasicInput {
private final RandomAccessFile file_;
private final long offset0_;
private final byte[] array_;
private final int bufsize_;
private final ByteBuffer buf_;
private final long filesize_;
private long bufOffset_;
public static final int DFLT_BUFSIZE = getDefaultBufferSize();
/**
* Constructs a BufferedRandomInput with a default sized buffer.
*
* @param file file
* @param offset0 offset into file of stream start
*/
public BufferedRandomInput( RandomAccessFile file, long offset0 )
throws IOException {
this( file, offset0, DFLT_BUFSIZE );
}
/**
* Constructs a BufferedRandomInput with a buffer of specified size.
*
* @param file file
* @param offset0 offset into file of stream start
* @param bufsize buffer size
*/
public BufferedRandomInput( RandomAccessFile file, long offset0,
int bufsize ) throws IOException {
file_ = file;
offset0_ = offset0;
bufsize_ = bufsize;
filesize_ = file.length();
array_ = new byte[ bufsize ];
buf_ = ByteBuffer.wrap( array_ );
buf_.clear();
bufOffset_ = offset0_;
buf_.limit( 0 );
assert getOffset() == 0;
assert buf_.remaining() == 0;
}
public byte readByte() throws IOException {
return getAssuredBuffer( 1 ).get();
}
public short readShort() throws IOException {
return getAssuredBuffer( 2 ).getShort();
}
public int readInt() throws IOException {
return getAssuredBuffer( 4 ).getInt();
}
public long readLong() throws IOException {
return getAssuredBuffer( 8 ).getLong();
}
public float readFloat() throws IOException {
return getAssuredBuffer( 4 ).getFloat();
}
public double readDouble() throws IOException {
return getAssuredBuffer( 8 ).getDouble();
}
public boolean isRandom() {
return true;
}
public void skip( long nbytes ) throws IOException {
seek( getOffset() + nbytes );
}
public long getOffset() {
return bufOffset_ + buf_.position() - offset0_;
}
public void seek( long offset ) throws IOException {
if ( offset < 0 ) {
throw new IOException( "Negative seek " + offset );
}
else if ( offset > filesize_ - offset0_ ) {
throw new EOFException( "Seek out of range " + offset );
}
else {
long bpos = offset - bufOffset_ + offset0_;
if ( bpos >= 0 && bpos <= buf_.limit() ) {
buf_.position( (int) bpos );
}
else {
bufOffset_ = offset + offset0_;
buf_.limit( 0 );
}
assert offset == getOffset();
}
}
public void close() throws IOException {
file_.close();
}
/**
* Returns a buffer positioned at the current read position
* guaranteed to be good for reading at least a given number of bytes.
*
* @param nbyte number of required bytes
* @return buffer positioned for read
*/
private ByteBuffer getAssuredBuffer( int nbyte ) throws IOException {
if ( buf_.remaining() >= nbyte ) {
return buf_;
}
else {
long bufoff = bufOffset_ + buf_.position();
file_.seek( bufoff );
int c = 0;
while ( c < nbyte ) {
int nb = file_.read( array_, c, bufsize_ - c );
if ( nb < 0 ) {
throw new EOFException();
}
c += nb;
}
bufOffset_ = bufoff;
buf_.position( 0 );
buf_.limit( c );
return buf_;
}
}
/**
* Returns the default buffer size.
* Currently returns whatever value BufferedInputStream uses.
*
* @return suitable size for a buffer
*/
private static int getDefaultBufferSize() {
final int[] sizer = new int[ 1 ];
new BufferedInputStream( new ByteArrayInputStream( new byte[ 0 ] ) ) {
{ sizer[ 0 ] = buf.length; }
};
return sizer[ 0 ];
}
}
......@@ -12,6 +12,7 @@ import java.util.List;
import java.util.logging.Logger;
import nom.tam.fits.BasicHDU;
import nom.tam.fits.FitsException;
import nom.tam.fits.FitsFactory;
import nom.tam.fits.FitsUtil;
import nom.tam.fits.Header;
import nom.tam.fits.HeaderCard;
......@@ -42,7 +43,7 @@ public class FitsConstants {
public static final String NDARRAY_PREFIX = "NDA_";
/** Image of end-of-header card. */
public static final HeaderCard END_CARD = new HeaderCard(
public static final HeaderCard END_CARD = createHeaderCard(
"END " +
" " );
......@@ -68,6 +69,9 @@ public class FitsConstants {
/** Maximum number of columns in standard FITS BINTABLE extension. */
public static final int MAX_NCOLSTD = 999;
/** Whether HIERARCH convention is used; true by default. */
public static boolean REQUIRE_HIERARCH = true;
private static final String[] extensions = new String[] {
".fits", ".fit", ".fts",
".FITS", ".FIT", ".FTS",
......@@ -89,10 +93,54 @@ public class FitsConstants {
return Collections.unmodifiableList( Arrays.asList( extensions ) );
}
/**
* Ensures that use of the HIERARCH convention when dealing with
* FITS headers is set to the state expected for STIL operation.
* If this involves changing the current setting, a warning is
* issued through the logging system.
*
* <p>The general idea is that this is only issued once per JVM.
* If other components are resetting the hierarch handling that
* might not be enough, but there's no mechanism to ensure that
* it stays set anyway.
*
* <p>The main reason this is necessary is that WideFits handling
* requires HIERARCH, but there is no per-call configuration of
* whether the convention is in use, so it has to be set on a
* global basis.
*/
public static void configureHierarch() {
boolean isReq = REQUIRE_HIERARCH;
if ( FitsFactory.getUseHierarch() != isReq ) {
logger.info( "Resetting FITS use of HIERARCH convention "
+ ! isReq + " -> " + isReq );
}
FitsFactory.setUseHierarch( isReq );
}
static String originCardName( int naxis ) {
return NDARRAY_ORIGIN + ( naxis + 1 );
}
/**
* Creates a Header instance which does not perform any unsolicited
* reordering of the header cards. Some versions of nom.tam.fits
* force the EXTEND keyword to go directly after the NAXISn keywords.
* That screws up specification of the FITS-plus magic number,
* which is expected to have VOTMETA directly after NAXISn.
* Headers created using this method will leave header cards in
* the order they are added.
*
* @return new empty header
* @see <a href="https://github.com/nom-tam-fits/nom-tam-fits/issues/113"
* >nom.tam.fits github discussion</a>
*/
public static Header createUnsortedHeader() {
Header hdr = new Header();
hdr.setHeaderSorter( null );
return hdr;
}
static int typeToBitpix( Type type ) {
if ( type == Type.BYTE ) {
return BasicHDU.BITPIX_BYTE;
......@@ -114,6 +162,16 @@ public class FitsConstants {
}
}
/**
* Create a HeaderCard from a FITS card image.
*
* @param cardImage the 80 character card image
* @return card object
*/
public static HeaderCard createHeaderCard( String cardImage ) {
return HeaderCard.create( cardImage );
}
/**
* Skips forward over a given number of HDUs in the supplied stream.
* If it reaches the end of the stream, it throws an IOException
......@@ -228,7 +286,8 @@ public class FitsConstants {
while ( need > 0 ) {
int len = strm.read( buffer, 80 - need, need );
if ( len <= 0 ) {
throw new TruncatedFileException();
throw new TruncatedFileException( "Stream stopped"
+ " mid-card" );
}
need -= len;
}
......@@ -242,7 +301,7 @@ public class FitsConstants {
}
String cbuf = new String( buffer );
HeaderCard fcard = new HeaderCard( cbuf );
HeaderCard fcard = createHeaderCard( cbuf );
if ( firstCard ) {
String key = fcard.getKey();
if ( key == null ||
......
......@@ -363,7 +363,8 @@ public class FitsNdxHandler
fchan.setEncoding( FitsConstants.WCS_ENCODING );
fchan.write( ndx.getAst() );
for ( Iterator it = fchan.iterator(); it.hasNext(); ) {
cardlist.add( new HeaderCard( (String) it.next() ) );
cardlist.add( FitsConstants
.createHeaderCard( (String) it.next() ) );
}
}
cards = (HeaderCard[]) cardlist.toArray( new HeaderCard[ 0 ] );
......@@ -538,7 +539,7 @@ public class FitsNdxHandler
if ( image.length() > 80 ) {
image = image.substring( 0, 80 );
}
return new HeaderCard( image );
return FitsConstants.createHeaderCard( image );
}
// implement HdxDocumentFactory
......
......@@ -10,8 +10,6 @@ import java.io.InputStream;
import java.io.RandomAccessFile;
import java.nio.channels.FileChannel;
import java.util.logging.Logger;
import nom.tam.util.BufferedFile;
import nom.tam.util.RandomAccess;
import uk.ac.starlink.util.Compression;
import uk.ac.starlink.util.DataSource;
import uk.ac.starlink.util.FileDataSource;
......@@ -155,13 +153,13 @@ public abstract class InputFactory implements Closeable {
};
}
else {
logger_.info( "Will read as BufferedFile: " + file
logger_.info( "Will read as BufferedRandomInput: " + file
+ " (avoid too much mapping on 32-bit OS" );
return new AbstractInputFactory( true ) {
public BasicInput createInput( boolean isSeq )
throws IOException {
BufferedFile bf = new BufferedFile( file.getPath(), "r" );
return new RandomAccessInput( bf, offset );
RandomAccessFile raf = new RandomAccessFile( file, "r" );
return new BufferedRandomInput( raf, offset );
}
public void close() {
}
......@@ -232,69 +230,4 @@ public abstract class InputFactory implements Closeable {
return isRandom_;
}
}
/**
* Adaptor from RandomAccess object to BasicInput object.
*/
private static class RandomAccessInput implements BasicInput {
private final RandomAccess stream_;
private final long offset_;
/**
* Constructor.
*
* @param stream input stream
* @param offset offset of data start
*/
public RandomAccessInput( RandomAccess stream, long offset )
throws IOException {
stream_ = stream;
offset_ = offset;
stream_.seek( offset );
}
public byte readByte() throws IOException {
return stream_.readByte();
}
public short readShort() throws IOException {
return stream_.readShort();
}
public int readInt() throws IOException {
return stream_.readInt();
}
public long readLong() throws IOException {
return stream_.readLong();
}
public float readFloat() throws IOException {
return stream_.readFloat();
}
public double readDouble() throws IOException {
return stream_.readDouble();
}
public void skip( long nbytes ) throws IOException {
seek( getOffset() + nbytes );
}
public boolean isRandom() {
return true;
}
public void seek( long offset ) throws IOException {
stream_.seek( offset_ + offset );
}
public long getOffset() {
return stream_.getFilePointer() - offset_;
}
public void close() throws IOException {
stream_.close();
}
}
}
......@@ -43,6 +43,7 @@ public class MappedFile extends AbstractArrayDataIO implements RandomAccess {
private final ByteBuffer niobuf_;
private int size_;
private long markPos_;
/**
* Constructs a MappedFile object from a byte buffer.
......@@ -113,6 +114,35 @@ public class MappedFile extends AbstractArrayDataIO implements RandomAccess {
return nskip;
}
public void skipAllBytes( long toSkip ) throws IOException {
if ( toSkip > 0 ) {
if ( toSkip <= niobuf_.remaining() ) {
int nskip = (int) toSkip;
assert nskip == toSkip;
niobuf_.position( niobuf_.position() + nskip );
}
else {
throw new EOFException();
}
}
}
public void skipAllBytes( int toSkip ) throws IOException {
skipAllBytes( (long) toSkip );
}
public boolean markSupported() {
return true;
}
public void mark( int readlimit ) {
markPos_ = getFilePointer();
}
public void reset() throws IOException {
seek( markPos_ );
}
protected byte get() throws IOException {
try {
return niobuf_.get();
......
......@@ -30,6 +30,7 @@ public class MultiMappedFile extends AbstractArrayDataIO
private final int blockBytes_;
private final int nblock_;
private int iblock_;
private long markPos_;
private static final Logger logger_ =
Logger.getLogger( "uk.ac.starlink.fits" );
......@@ -91,6 +92,21 @@ public class MultiMappedFile extends AbstractArrayDataIO
return ns;
}
public void skipAllBytes( long nskip ) throws IOException {
if ( nskip > 0 ) {
if ( nskip <= remaining() ) {
seek( getFilePointer() + nskip );
}
else {
throw new EOFException();
}
}
}
public void skipAllBytes( int toSkip ) throws IOException {
skipAllBytes( (long) toSkip );
}
public long getFilePointer() {
try {
return (long) iblock_ * blockBytes_
......@@ -103,7 +119,19 @@ public class MultiMappedFile extends AbstractArrayDataIO
public int skipBytes( int toSkip ) throws IOException {
return toInt( skip( (long) toSkip ) );
};
}
public boolean markSupported() {
return true;
}
public void mark( int readlimit ) {
markPos_ = getFilePointer();
}
public void reset() throws IOException {
seek( markPos_ );
}
protected byte get() throws IOException {
try {
......
......@@ -153,7 +153,7 @@ class WritableFitsArrayImpl implements ArrayImpl {
}
/* Add termination record. */
cardlist.add( new HeaderCard( "END" ) );
cardlist.add( FitsConstants.END_CARD );
}
catch( HeaderCardException e ) {
throw (IOException) new IOException( e.getMessage() )
......
......@@ -57,6 +57,12 @@ public class BasicInputTest extends TestCase {
new FileInputStream( file ) ) ) );
exerciseInput( seqOffInput( off1,
new BufferedFile( file.getPath() ) ) );
exerciseInput( new BufferedRandomInput( new RandomAccessFile( file,
"r" ),
off1 ) );
exerciseInput( new BufferedRandomInput( new RandomAccessFile( file,
"r" ),
off1, 29 ) );
exerciseInput( new SimpleMappedInput( chan, off1, leng1, "test" ) );
exerciseInput( BlockMappedInput
.createInput( chan, off1, leng1, "test", isiz + 3, 0 ) );
......
package uk.ac.starlink.fits;
import junit.framework.TestCase;
import nom.tam.fits.HeaderCard;
import nom.tam.fits.HeaderCardException;
public class TamfitsTest extends TestCase {
public void testString() throws Exception {
// These tests test bugfixes made to the starjava fork
// of nom.tam.fits. If they fail on other nom.tam.fits
// versions, those versions should be fixed.
String name = "DUMMY";
String txt = repeat( "1234567890", 8 );
String comm = null;
for ( int i = 0; i < 69; i++ ) {
HeaderCard card =
new HeaderCard( name, txt.substring( 0, i ), comm );
checkStringCard( card );
}
assertEquals( '\'', new HeaderCard( name, txt.substring( 0, 68 ), comm )
.toString().charAt( 79 ) );
try {
new HeaderCard( name, txt.substring( 0, 69 ), comm );
fail();
}
catch ( HeaderCardException e ) {
// ok
}
{
String qtxt = repeat( "C'thalpa ", 6 ) + "12345678";
HeaderCard qcard = new HeaderCard( name, qtxt, comm );
checkStringCard( qcard );
assertEquals( 80, qcard.toString().length() );
try {
new HeaderCard( name, qtxt + "X", comm );
fail();
}
catch ( HeaderCardException e ) {
// ok
}
}
String atxt = "Cy\u00e4egha\u2207";
{
HeaderCard acard = new HeaderCard( name, atxt, comm );
checkStringCard( acard );
}
{
String ltxt = repeat( "1234567890", 6 ) + atxt;
HeaderCard lcard = new HeaderCard( name, ltxt, comm );
checkStringCard( lcard );
assertEquals( 80, lcard.toString().length() );
}
}
private void checkStringCard( HeaderCard card ) {
String image = card.toString();
assertEquals( 80, image.length() );
assertEquals( '=', image.charAt( 8 ) );
assertEquals( ' ', image.charAt( 9 ) );
assertEquals( '\'', image.charAt( 10 ) );
assertEquals( '\'', image.charAt( image.trim().length() - 1 ) );
for ( int ic = 0; ic < 80; ic++ ) {
char c = image.charAt( ic );
assertTrue( "Disallowed character: " + c + " in " + image,
c >= 0x20 && c <= 0x7e );
}
}
private static String repeat( String txt, int count ) {
StringBuffer sbuf = new StringBuffer( txt.length() * count );
for ( int i = 0; i < count; i++ ) {
sbuf.append( txt );
}
return sbuf.toString();
}
}
......@@ -26,6 +26,7 @@ public class WideTest extends TestCase {
.setLevel( Level.SEVERE );
Logger.getLogger( "uk.ac.starlink.table" )
.setLevel( Level.WARNING );
FitsConstants.configureHierarch();
}
// AlphaWideFits is a historical relic - this test could be removed
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment