Changeset 1757a2c


Ignore:
Timestamp:
Jan 11, 2011 4:40:37 PM (9 years ago)
Author:
dev <dev@…>
Branches:
master
Children:
7289c89
Parents:
16509e59 (diff), a51ae64e (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.
Message:

merge of '4e478ff8494e308808d6df2c47e368ac3aeb61aa'

and '51ae655150adb03f61e4ead408e1e7d827661dc7'

Files:
11 edited

Legend:

Unmodified
Added
Removed
  • build.xml

    r16509e59 r1757a2c  
    756756    <target name="findbugs" depends="build2">
    757757        <echo message="Starting findbugs, this will take a while..." />
    758         <exec executable="nice">
     758        <exec executable="nice" failonerror="true">
    759759            <arg value="findbugs"/>
    760760            <arg value="-textui"/>
  • core/java/src/net/i2p/crypto/SHA256Generator.java

    r16509e59 r1757a2c  
    2424    }
    2525   
    26     /** Calculate the SHA-256 has of the source
     26    /**
     27     * Calculate the SHA-256 hash of the source and cache the result.
    2728     * @param source what to hash
    2829     * @return hash of the source
     
    3132        return calculateHash(source, 0, source.length);
    3233    }
     34
     35    /**
     36     * Calculate the hash and cache the result.
     37     */
    3338    public final Hash calculateHash(byte[] source, int start, int len) {
    3439        Sha256Standalone digest = acquireGnu();
     
    4045    }
    4146   
     47    /**
     48     * Use this if you only need the data, not a Hash object.
     49     * Does not cache.
     50     */
    4251    public final void calculateHash(byte[] source, int start, int len, byte out[], int outOffset) {
    4352        Sha256Standalone digest = acquireGnu();
  • core/java/src/net/i2p/data/DataHelper.java

    r16509e59 r1757a2c  
    3232import java.util.Collection;
    3333import java.util.Collections;
     34import java.util.Comparator;
    3435import java.util.Date;
    3536import java.util.Iterator;
     
    10411042
    10421043    /**
    1043      *  Sort based on the Hash of the DataStructure
     1044     *  Sort based on the Hash of the DataStructure.
    10441045     *  Warning - relatively slow.
    1045      *  Only used by RouterInfo
    1046      *  Why? Just because it has to be consistent so signing will work?
     1046     *  WARNING - this sort order must be consistent network-wide, so while the order is arbitrary,
     1047     *  it cannot be changed.
     1048     *  Why? Just because it has to be consistent so signing will work.
    10471049     *  How to spec as returning the same type as the param?
     1050     *  DEPRECATED - Only used by RouterInfo.
    10481051     */
    10491052    public static List<? extends DataStructure> sortStructures(Collection<? extends DataStructure> dataStructures) {
    10501053        if (dataStructures == null) return Collections.EMPTY_LIST;
    1051         ArrayList<DataStructure> rv = new ArrayList(dataStructures.size());
    1052         TreeMap<String, DataStructure> tm = new TreeMap();
    1053         for (DataStructure struct : dataStructures) {
    1054             tm.put(struct.calculateHash().toString(), struct);
    1055         }
    1056         for (DataStructure struct : tm.values()) {
    1057             rv.add(struct);
    1058         }
     1054
     1055        // This used to use Hash.toString(), which is insane, since a change to toString()
     1056        // would break the whole network. Now use Hash.toBase64().
     1057        // Note that the Base64 sort order is NOT the same as the raw byte sort order,
     1058        // despite what you may read elsewhere.
     1059
     1060        //ArrayList<DataStructure> rv = new ArrayList(dataStructures.size());
     1061        //TreeMap<String, DataStructure> tm = new TreeMap();
     1062        //for (DataStructure struct : dataStructures) {
     1063        //    tm.put(struct.calculateHash().toString(), struct);
     1064        //}
     1065        //for (DataStructure struct : tm.values()) {
     1066        //    rv.add(struct);
     1067        //}
     1068        ArrayList<DataStructure> rv = new ArrayList(dataStructures);
     1069        Collections.sort(rv, new DataStructureComparator());
    10591070        return rv;
     1071    }
     1072
     1073    /**
     1074     * See sortStructures() comments.
     1075     * @since 0.8.3
     1076     */
     1077    private static class DataStructureComparator implements Comparator<DataStructure> {
     1078        public int compare(DataStructure l, DataStructure r) {
     1079            return l.calculateHash().toBase64().compareTo(r.calculateHash().toBase64());
     1080        }
    10601081    }
    10611082
  • core/java/src/net/i2p/data/RouterAddress.java

    r16509e59 r1757a2c  
    132132    }
    133133   
    134     /** the style should be sufficient, for speed */
     134    /**
     135     * Just use style and hashCode for speed (expiration is always null).
     136     * If we add multiple addresses of the same style, this may need to be changed.
     137     */
    135138    @Override
    136139    public int hashCode() {
    137         return DataHelper.hashCode(_transportStyle);
     140        return DataHelper.hashCode(_transportStyle) ^ _cost;
    138141    }
    139142   
  • core/java/src/net/i2p/data/RouterInfo.java

    r16509e59 r1757a2c  
    1515import java.io.OutputStream;
    1616import java.util.Collection;
     17import java.util.Collections;
    1718import java.util.Date;
    1819import java.util.HashSet;
     
    4041    private volatile long _published;
    4142    private final Set<RouterAddress> _addresses;
    42     private final Set<Hash> _peers;
     43    /** may be null to save memory, no longer final */
     44    private Set<Hash> _peers;
    4345    private /* FIXME final FIXME */ Properties _options;
    4446    private volatile boolean _validated;
     
    4850    private volatile int _hashCode;
    4951    private volatile boolean _hashCodeInitialized;
     52    /** should we cache the byte and string versions _byteified ? **/
     53    private boolean _shouldCache;
     54    /** maybe we should check if we are floodfill? */
     55    private static final boolean CACHE_ALL = Runtime.getRuntime().maxMemory() > 128*1024*1024l;
    5056
    5157    public static final String PROP_NETWORK_ID = "netId";
     
    5965    public RouterInfo() {
    6066        _addresses = new HashSet(2);
    61         _peers = new HashSet(0);
    6267        _options = new OrderedProperties();
    6368    }
     
    7176        setOptions(old.getOptions());
    7277        setSignature(old.getSignature());
     78        // copy over _byteified?
    7379    }
    7480
     
    106112        _identity = ident;
    107113        resetCache();
     114        // We only want to cache the bytes for our own RI, which is frequently written.
     115        // To cache for all RIs doubles the RI memory usage.
     116        // setIdentity() is only called when we are creating our own RI.
     117        // Otherwise, the data is populated with readBytes().
     118        _shouldCache = true;
    108119    }
    109120
     
    160171     */
    161172    public Set<Hash> getPeers() {
     173        if (_peers == null)
     174            return Collections.EMPTY_SET;
    162175        return _peers;
    163176    }
     
    170183     */
    171184    public void setPeers(Set<Hash> peers) {
     185        if (peers == null || peers.isEmpty()) {
     186            _peers = null;
     187            return;
     188        }
     189        if (_peers == null)
     190            _peers = new HashSet(2);
    172191        synchronized (_peers) {
    173192            _peers.clear();
    174             if (peers != null) _peers.addAll(peers);
     193            _peers.addAll(peers);
    175194        }
    176195        resetCache();
     
    224243        if (_identity == null) throw new DataFormatException("Router identity isn't set? wtf!");
    225244        if (_addresses == null) throw new DataFormatException("Router addressess isn't set? wtf!");
    226         if (_peers == null) throw new DataFormatException("Router peers isn't set? wtf!");
    227245        if (_options == null) throw new DataFormatException("Router options isn't set? wtf!");
    228246
     
    240258                Collection<RouterAddress> addresses = _addresses;
    241259                if (sz > 1)
     260                    // WARNING this sort algorithm cannot be changed, as it must be consistent
     261                    // network-wide. The signature is not checked at readin time, but only
     262                    // later, and the addresses are stored in a Set, not a List.
    242263                    addresses = (Collection<RouterAddress>) DataHelper.sortStructures(addresses);
    243264                for (RouterAddress addr : addresses) {
     
    249270            //         method of trusted links, which isn't implemented in the router
    250271            //         at the moment, and may not be later.
    251             // fixme to reduce objects - allow _peers == null
    252             int psz = _peers.size();
     272            int psz = _peers == null ? 0 : _peers.size();
    253273            DataHelper.writeLong(out, 1, psz);
    254274            if (psz > 0) {
    255275                Collection<Hash> peers = _peers;
    256276                if (psz > 1)
     277                    // WARNING this sort algorithm cannot be changed, as it must be consistent
     278                    // network-wide. The signature is not checked at readin time, but only
     279                    // later, and the hashes are stored in a Set, not a List.
    257280                    peers = (Collection<Hash>) DataHelper.sortStructures(peers);
    258281                for (Hash peerHash : peers) {
     
    267290        long after = Clock.getInstance().now();
    268291        _log.debug("getBytes()  took " + (after - before) + "ms");
    269         _byteified = data;
     292        if (CACHE_ALL || _shouldCache)
     293            _byteified = data;
    270294        return data;
    271295    }
     
    467491        }
    468492        int numPeers = (int) DataHelper.readLong(in, 1);
    469         for (int i = 0; i < numPeers; i++) {
    470             Hash peerIdentityHash = new Hash();
    471             peerIdentityHash.readBytes(in);
    472             _peers.add(peerIdentityHash);
     493        if (numPeers == 0) {
     494            _peers = null;
     495        } else {
     496            _peers = new HashSet(numPeers);
     497            for (int i = 0; i < numPeers; i++) {
     498                Hash peerIdentityHash = new Hash();
     499                peerIdentityHash.readBytes(in);
     500                _peers.add(peerIdentityHash);
     501            }
    473502        }
    474503        _options = DataHelper.readProperties(in);
     
    505534               && DataHelper.eq(_addresses, info.getAddresses())
    506535               && DataHelper.eq(_options, info.getOptions())
    507                && DataHelper.eq(_peers, info.getPeers());
     536               && DataHelper.eq(getPeers(), info.getPeers());
    508537    }
    509538   
     
    531560            buf.append("\n\t\tAddress: ").append(addr);
    532561        }
    533         Set peers = _peers; // getPeers()
     562        Set peers = getPeers();
    534563        buf.append("\n\tPeers: #: ").append(peers.size());
    535564        for (Iterator iter = peers.iterator(); iter.hasNext();) {
  • history.txt

    r16509e59 r1757a2c  
     12011-01-09 zzz
     2    * DataHelper: Speed up and annotate sortStructures()
     3    * Data Structures: More caching improvements, don't cache where we shouldn't
     4    * NetDB: Don't rescan netDb directory unless changed,
     5             to reduce Hash cache thrash (backport from test4)
     6    * RouterInfo:
     7      - Don't cache byteified data by default, to save ~1.5 MB
     8      - Don't create empty peers Set, to save ~100KB
     9
    1102011-01-07 zzz
    211    * Data Structures: More caching
  • router/java/src/net/i2p/data/i2np/I2NPMessageImpl.java

    r16509e59 r1757a2c  
    2121import net.i2p.data.Hash;
    2222import net.i2p.util.Log;
     23import net.i2p.util.SimpleByteCache;
    2324
    2425/**
     
    7374     *  readMessage in the implemented message type
    7475     *
     76     *<pre>
    7577     *  Specifically:
    7678     *    1 byte type (if caller didn't read already, as specified by the type param
     
    8082     *    1 byte checksum
    8183     *    size bytes of payload (read by readMessage() in implementation)
     84     *</pre>
    8285     *
    8386     *  @param type the message type or -1 if we should read it here
    8487     *  @param buffer temp buffer to use
     88     *  @return total length of the message
    8589     */
    8690    public int readBytes(InputStream in, int type, byte buffer[]) throws I2NPMessageException, IOException {
     
    111115            }
    112116           
    113             Hash calc = _context.sha().calculateHash(buffer, 0, size);
     117            byte[] calc = SimpleByteCache.acquire(Hash.HASH_LENGTH);
     118            _context.sha().calculateHash(buffer, 0, size, calc, 0);
    114119            //boolean eq = calc.equals(h);
    115             boolean eq = DataHelper.eq(checksum, 0, calc.getData(), 0, CHECKSUM_LENGTH);
     120            boolean eq = DataHelper.eq(checksum, 0, calc, 0, CHECKSUM_LENGTH);
     121            SimpleByteCache.release(calc);
    116122            if (!eq)
    117123                throw new I2NPMessageException("Hash does not match for " + getClass().getName());
     
    124130            //if (time > 50)
    125131            //    _context.statManager().addRateData("i2np.readTime", time, time);
    126             return size + Hash.HASH_LENGTH + 1 + 4 + DataHelper.DATE_LENGTH;
     132            return CHECKSUM_LENGTH + 1 + 2 + 4 + DataHelper.DATE_LENGTH + size;
    127133        } catch (DataFormatException dfe) {
    128134            throw new I2NPMessageException("Error reading the message header", dfe);
    129135        }
    130136    }
     137
     138    /**
     139     *  Read the header, then read the rest into buffer, then call
     140     *  readMessage in the implemented message type
     141     *
     142     *<pre>
     143     *  Specifically:
     144     *    1 byte type (if caller didn't read already, as specified by the type param
     145     *    4 byte ID
     146     *    8 byte expiration
     147     *    2 byte size
     148     *    1 byte checksum
     149     *    size bytes of payload (read by readMessage() in implementation)
     150     *</pre>
     151     *
     152     *  @param type the message type or -1 if we should read it here
     153     *  @return total length of the message
     154     */
    131155    public int readBytes(byte data[], int type, int offset) throws I2NPMessageException, IOException {
    132156        int cur = offset;
     
    154178                                           + " wanted=" + size + "]: " + getClass().getName());
    155179
    156         Hash calc = _context.sha().calculateHash(data, cur, size);
    157         //boolean eq = calc.equals(h);
    158         boolean eq = DataHelper.eq(hdata, 0, calc.getData(), 0, CHECKSUM_LENGTH);
     180        byte[] calc = SimpleByteCache.acquire(Hash.HASH_LENGTH);
     181        _context.sha().calculateHash(data, cur, size, calc, 0);
     182        boolean eq = DataHelper.eq(hdata, 0, calc, 0, CHECKSUM_LENGTH);
     183        SimpleByteCache.release(calc);
    159184        if (!eq)
    160185            throw new I2NPMessageException("Hash does not match for " + getClass().getName());
     
    232257            int writtenLen = writeMessageBody(buffer, prefixLen);
    233258            int payloadLen = writtenLen - prefixLen;
    234             Hash h = _context.sha().calculateHash(buffer, prefixLen, payloadLen);
     259            byte[] h = SimpleByteCache.acquire(Hash.HASH_LENGTH);
     260            _context.sha().calculateHash(buffer, prefixLen, payloadLen, h, 0);
    235261
    236262            int off = 0;
     
    243269            DataHelper.toLong(buffer, off, 2, payloadLen);
    244270            off += 2;
    245             System.arraycopy(h.getData(), 0, buffer, off, CHECKSUM_LENGTH);
     271            System.arraycopy(h, 0, buffer, off, CHECKSUM_LENGTH);
     272            SimpleByteCache.release(h);
    246273
    247274            //long time = _context.clock().now() - start;
  • router/java/src/net/i2p/router/RouterVersion.java

    r16509e59 r1757a2c  
    1919    public final static String ID = "Monotone";
    2020    public final static String VERSION = CoreVersion.VERSION;
    21     public final static long BUILD = 9;
     21    public final static long BUILD = 10;
    2222
    2323    /** for example "-test" */
  • router/java/src/net/i2p/router/networkdb/kademlia/KademliaNetworkDatabaseFacade.java

    r16509e59 r1757a2c  
    248248       
    249249        _kb = new KBucketSet(_context, ri.getIdentity().getHash());
    250         _ds = new PersistentDataStore(_context, dbDir, this);
     250        try {
     251            _ds = new PersistentDataStore(_context, dbDir, this);
     252        } catch (IOException ioe) {
     253            throw new RuntimeException("Unable to initialize netdb storage", ioe);
     254        }
    251255        //_ds = new TransientDataStore();
    252256//        _exploreKeys = new HashSet(64);
  • router/java/src/net/i2p/router/networkdb/kademlia/PersistentDataStore.java

    r16509e59 r1757a2c  
    4040 */
    4141class PersistentDataStore extends TransientDataStore {
    42     private Log _log;
    43     private String _dbDir;
    44     private KademliaNetworkDatabaseFacade _facade;
    45     private Writer _writer;
    46     private ReadJob _readJob;
     42    private final Log _log;
     43    private final File _dbDir;
     44    private final KademliaNetworkDatabaseFacade _facade;
     45    private final Writer _writer;
     46    private final ReadJob _readJob;
    4747    private boolean _initialized;
    4848   
    4949    private final static int READ_DELAY = 60*1000;
    5050   
    51     public PersistentDataStore(RouterContext ctx, String dbDir, KademliaNetworkDatabaseFacade facade) {
     51    /**
     52     *  @param dbDir relative path
     53     */
     54    public PersistentDataStore(RouterContext ctx, String dbDir, KademliaNetworkDatabaseFacade facade) throws IOException {
    5255        super(ctx);
    5356        _log = ctx.logManager().getLog(PersistentDataStore.class);
    54         _dbDir = dbDir;
     57        _dbDir = getDbDir(dbDir);
    5558        _facade = facade;
    5659        _readJob = new ReadJob();
     
    8083    public void restart() {
    8184        super.restart();
    82         _dbDir = _facade.getDbDir();
    8385    }
    8486   
     
    161163                _log.info("Removing key " + _key /* , getAddedBy() */);
    162164            try {
    163                 File dbDir = getDbDir();
    164                 removeFile(_key, dbDir);
     165                removeFile(_key, _dbDir);
    165166            } catch (IOException ioe) {
    166167                _log.error("Error removing key " + _key, ioe);
     
    237238                if (key != null) {
    238239                    if (data != null) {
    239                         write(key, data);
     240                        // synch with the reader job
     241                        synchronized (_dbDir) {
     242                            write(key, data);
     243                        }
    240244                        data = null;
    241245                    }
     
    279283        try {
    280284            String filename = null;
    281             File dbDir = getDbDir();
    282285
    283286            if (data instanceof LeaseSet)
     
    288291                throw new IOException("We don't know how to write objects of type " + data.getClass().getName());
    289292
    290             dbFile = new File(dbDir, filename);
     293            dbFile = new File(_dbDir, filename);
    291294            long dataPublishDate = getPublishDate(data);
    292295            if (dbFile.lastModified() < dataPublishDate) {
     
    327330    private class ReadJob extends JobImpl {
    328331        private boolean _alreadyWarned;
     332        private long _lastModified;
     333
    329334        public ReadJob() {
    330335            super(PersistentDataStore.this._context);
    331336            _alreadyWarned = false;
    332337        }
     338
    333339        public String getName() { return "DB Read Job"; }
     340
    334341        public void runJob() {
    335             _log.info("Rereading new files");
    336             readFiles();
     342            // check directory mod time to save a lot of object churn in scanning all the file names
     343            long lastMod = _dbDir.lastModified();
     344            if (lastMod > _lastModified) {
     345                _lastModified = lastMod;
     346                _log.info("Rereading new files");
     347                // synch with the writer job
     348                synchronized (_dbDir) {
     349                    readFiles();
     350                }
     351            }
    337352            requeue(READ_DELAY);
    338353        }
     
    344359        private void readFiles() {
    345360            int routerCount = 0;
    346             try {
    347                 File dbDir = getDbDir();
    348                 File routerInfoFiles[] = dbDir.listFiles(RouterInfoFilter.getInstance());
     361
     362                File routerInfoFiles[] = _dbDir.listFiles(RouterInfoFilter.getInstance());
    349363                if (routerInfoFiles != null) {
    350364                    routerCount += routerInfoFiles.length;
     
    361375                    }
    362376                }
    363             } catch (IOException ioe) {
    364                 _log.error("Error reading files in the db dir", ioe);
    365             }
    366377           
    367378            if (!_alreadyWarned) {
     
    443454   
    444455   
    445     private File getDbDir() throws IOException {
    446         File f = new SecureDirectory(_context.getRouterDir(), _dbDir);
     456    private File getDbDir(String dbDir) throws IOException {
     457        File f = new SecureDirectory(_context.getRouterDir(), dbDir);
    447458        if (!f.exists()) {
    448459            boolean created = f.mkdirs();
  • router/java/src/net/i2p/router/tunnel/FragmentHandler.java

    r16509e59 r1757a2c  
    1616import net.i2p.util.ByteCache;
    1717import net.i2p.util.Log;
     18import net.i2p.util.SimpleByteCache;
    1819import net.i2p.util.SimpleTimer;
    1920
     
    242243            _log.debug("endpoint IV: " + Base64.encode(preV, validLength - HopProcessor.IV_LENGTH, HopProcessor.IV_LENGTH));
    243244       
    244         Hash v = _context.sha().calculateHash(preV, 0, validLength);
     245        byte[] v = SimpleByteCache.acquire(Hash.HASH_LENGTH);
     246        _context.sha().calculateHash(preV, 0, validLength, v, 0);
    245247        _validateCache.release(ba);
    246248       
    247         boolean eq = DataHelper.eq(v.getData(), 0, preprocessed, offset + HopProcessor.IV_LENGTH, 4);
     249        boolean eq = DataHelper.eq(v, 0, preprocessed, offset + HopProcessor.IV_LENGTH, 4);
    248250        if (!eq) {
    249251            if (_log.shouldLog(Log.WARN)) {
    250252                _log.warn("Corrupt tunnel message - verification fails: " + Base64.encode(preprocessed, offset+HopProcessor.IV_LENGTH, 4)
    251                            + " != " + Base64.encode(v.getData(), 0, 4));
     253                           + " != " + Base64.encode(v, 0, 4));
    252254                _log.warn("No matching endpoint: # pad bytes: " + (paddingEnd-(HopProcessor.IV_LENGTH+4)-1)
    253255                           + " offset=" + offset + " length=" + length + " paddingEnd=" + paddingEnd + ' '
     
    255257            }
    256258        }
     259        SimpleByteCache.release(v);
    257260       
    258261        if (eq) {
Note: See TracChangeset for help on using the changeset viewer.