Changeset e117e33


Ignore:
Timestamp:
Dec 11, 2011 10:59:37 PM (9 years ago)
Author:
zzz <zzz@…>
Branches:
master
Children:
1e5afa85
Parents:
8448001
Message:
  • ReusableGZIPInputStream: Fix 3 major bugs, all present since 2005:
    • Payloads an exact multiple of 512 bytes failed to decompress
    • Data at the end of the stream could be lost
    • read() returned -1 when the data was 0xff
Files:
4 edited

Legend:

Unmodified
Added
Removed
  • core/java/src/net/i2p/util/LookaheadInputStream.java

    r8448001 re117e33  
    1818    private static final InputStream _fakeInputStream = new ByteArrayInputStream(new byte[0]);
    1919   
     20    /**
     21     *  Configure a stream that hides a number of bytes from the reader.
     22     *  The last n bytes will never be available from read(),
     23     *  they can only be obtained from getFooter().
     24     *
     25     *  initialize() MUST be called before doing any read() calls.
     26     *
     27     *  @param lookaheadSize how many bytes to hide
     28     */
    2029    public LookaheadInputStream(int lookaheadSize) {
    2130        super(_fakeInputStream);
     
    2534    public boolean getEOFReached() { return _eofReached; }
    2635       
    27     /** blocking! */
     36    /**
     37     *  Start the LookaheadInputStream with the given input stream.
     38     *  Resets everything if the LookaheadInputStream was previously used.
     39     *  WARNING - blocking until lookaheadSize bytes are read!
     40     */
    2841    public void initialize(InputStream src) throws IOException {
    2942        in = src;
     
    4861        }
    4962        int rv = _footerLookahead[0];
     63        // FIXME use an index!!!!!!!!!!!!
    5064        System.arraycopy(_footerLookahead, 1, _footerLookahead, 0, _footerLookahead.length-1);
    5165        _footerLookahead[_footerLookahead.length-1] = (byte)c;
  • core/java/src/net/i2p/util/ResettableGZIPInputStream.java

    r8448001 re117e33  
    22
    33import java.io.ByteArrayInputStream;
     4import java.io.FilterInputStream;
    45import java.io.IOException;
    56import java.io.InputStream;
     
    2021    private static final int FOOTER_SIZE = 8; // CRC32 + ISIZE
    2122    private static final boolean DEBUG = false;
    22     /** keep a typesafe copy of (LookaheadInputStream)in */
     23    /** See below for why this is necessary */
     24    private final ExtraByteInputStream _extraByteInputStream;
     25    /** keep a typesafe copy of this */
    2326    private final LookaheadInputStream _lookaheadStream;
     27    private static final byte[] _oneDummyByte = new byte[1];
     28    private final InputStream _sequenceStream = null;
    2429    private final CRC32 _crc32;
    2530    private final byte _buf1[] = new byte[1];
     
    3237     */
    3338    public ResettableGZIPInputStream() {
    34         super(new LookaheadInputStream(FOOTER_SIZE), new Inflater(true));
    35         _lookaheadStream = (LookaheadInputStream)in;
     39        // compressedStream ->
     40        //   LookaheadInputStream that removes last 8 bytes ->
     41        //     ExtraByteInputStream that adds 1 byte ->
     42        //       InflaterInputStream
     43        // See below for why this is necessary
     44        super(new ExtraByteInputStream(new LookaheadInputStream(FOOTER_SIZE)),
     45              new Inflater(true));
     46        _extraByteInputStream = (ExtraByteInputStream)in;
     47        _lookaheadStream = (LookaheadInputStream)_extraByteInputStream.getInputStream();
    3648        _crc32 = new CRC32();
    3749    }
     
    5668        _crc32.reset();
    5769        _buf1[0] = 0x0;
     70        _extraByteInputStream.reset();
    5871        // blocking call to read the footer/lookahead, and use the compressed
    5972        // stream as the source for further lookahead bytes
     
    6679    @Override
    6780    public int read() throws IOException {
    68         if (_complete) {
    69             // shortcircuit so the inflater doesn't try to refill
    70             // with the footer's data (which would fail, causing ZLIB err)
    71             return -1;
    72         }
    7381        int read = read(_buf1, 0, 1);
    7482        if (read == -1)
    7583            return -1;
    76         else
    77             return _buf1[0];
     84        return _buf1[0] & 0xff;
    7885    }
    7986   
     
    8390    }
    8491
     92    /**
     93     *
     94     */
    8595    @Override
    8696    public int read(byte buf[], int off, int len) throws IOException {
     
    96106        } else {
    97107            _crc32.update(buf, off, read);
    98             if (_lookaheadStream.getEOFReached()) {
     108            // NO, we can't do use getEOFReached here
     109            // 1) Just because the lookahead stream has hit EOF doesn't mean
     110            //    that the inflater has given us all the data yet,
     111            //    this would cause data loss at the end
     112            //if (_lookaheadStream.getEOFReached()) {
     113            if (inf.finished()) {
    99114                verifyFooter();
    100115                inf.reset(); // so it doesn't bitch about missing data...
     
    273288            c = in.read();
    274289            if (c == -1) throw new IOException("EOF reading the CRC16");
     290        }
     291    }
     292   
     293    /**
     294     *  Essentially a SequenceInputStream(in, new ByteArrayInputStream(new byte[1])),
     295     *  except that this is resettable.
     296     *
     297     *  Unsupported:
     298     *    - available() doesn't include the extra byte
     299     *    - skip() doesn't skip the extra byte
     300     *
     301     *  Why? otherwise the inflater finished() is wrong when the compressed payload
     302     *  (in between the 10 byte header and the 8 byte footer) is a multiple of 512 bytes,
     303     *  which caused read(buf, off, len) above to fail.
     304     *  Happened every time with 1042 byte compressed router infos, for example.
     305     *
     306     *  Details:
     307     *
     308     *  Warning with Inflater nowrap = true:
     309     *
     310     *     "Note: When using the 'nowrap' option it is also necessary to provide an extra "dummy" byte as input.
     311     *      This is required by the ZLIB native library in order to support certain optimizations."
     312     *
     313     *     http://docs.oracle.com/javase/1.5.0/docs/api/java/util/zip/Inflater.html
     314     *
     315     *  This is for sure:
     316     *
     317     *     "This is not nearly specific enough to be useful.  Where in the compressed byte array is the
     318     *      extra 'dummy' byte" expected?  What is it to contain? When calling setInput() is the 'len'
     319     *      argument incremented to include the dummy byte or not?"
     320     *
     321     *     http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4795299
     322     *
     323     *  This is useless:
     324     *
     325     *     http://www.java-forums.org/new-java/38604-decompress-un-gzip-byte.html
     326     *
     327     *  This seems to be the definitive answer:
     328     *
     329     *     "The fix simply involves copying the byte array and tacking a single null byte on to the end."
     330     *
     331     *     http://code.google.com/p/google-apps-sso-sample/issues/detail?id=8
     332     *
     333     */
     334    private static class ExtraByteInputStream extends FilterInputStream {
     335        private static final byte DUMMY = 0;
     336        private boolean _extraSent;
     337
     338        public ExtraByteInputStream(InputStream in) {
     339            super(in);
     340        }
     341
     342        @Override
     343        public int read() throws IOException {
     344            if (_extraSent)
     345                return -1;
     346            int rv = in.read();
     347            if (rv >= 0)
     348                return rv;
     349            _extraSent = true;
     350            return DUMMY;
     351        }
     352
     353        @Override
     354        public int read(byte buf[], int off, int len) throws IOException {
     355            if (len == 0)
     356                return 0;
     357            if (_extraSent)
     358                return -1;
     359            int rv = in.read(buf, off, len);
     360            if (rv >= 0)
     361                return rv;
     362            _extraSent = true;
     363            buf[off] = DUMMY;
     364            return 1;
     365        }
     366
     367        @Override
     368        public void close() throws IOException {
     369            _extraSent = false;
     370            in.close();
     371        }
     372
     373        /** does NOT call in.reset() */
     374        @Override
     375        public void reset() {
     376            _extraSent = false;
     377        }
     378
     379        public InputStream getInputStream() {
     380            return in;
    275381        }
    276382    }
  • history.txt

    r8448001 re117e33  
     12011-12-12 zzz
     2  * ExploreJob: Tweaks to handle DatabaseLookupMessage changes
     3  * I2NP:
     4    - Deprecate unused stream methods and I2NPMessageReader since
     5      all transports provide encapsulation.
     6    - Don't throw IOE from byte array methods
     7    - Use cached null cert in GarlicClove
     8    - Add method to limit size of buffer to read
     9    - Don't check checksum at input, in most cases
     10    - Reuse checksum at output, for unomodified pass-through messages
     11      (but recalculating it now and logging on a mismatch for testing)
     12    - Fix DatabaseLookupMessage to internally store the don't include peers as
     13      a List, not a Set, so it doesn't get reordered and break the checksum
     14    - Log cleanup
     15  * NTCP:
     16    - Zero-copy and limit size when handing buffer to I2NP
     17    - Log hex dump message on I2NPMessageException, like in SSU
     18    - Don't close connection on I2NPMessageException
     19  * PortMapper: New service for registering application ports in the context
     20  * ReusableGZIPInputStream: Fix 3 major bugs, all present since 2005:
     21    - Payloads an exact multiple of 512 bytes failed to decompress
     22    - Data at the end of the stream could be lost
     23    - read() returned -1 when the data was 0xff
     24  * SearchState: generics and cleanups
     25
    1262011-12-11 sponge
    227  * HTTP server tunnel, use log.WARN for 3 first minutes. (closes #460)
  • router/java/src/net/i2p/router/RouterVersion.java

    r8448001 re117e33  
    1919    public final static String ID = "Monotone";
    2020    public final static String VERSION = CoreVersion.VERSION;
    21     public final static long BUILD = 19;
     21    public final static long BUILD = 20;
    2222
    2323    /** for example "-test" */
Note: See TracChangeset for help on using the changeset viewer.