Changeset 16bec08f


Ignore:
Timestamp:
May 3, 2010 4:44:06 PM (11 years ago)
Author:
walking <walking@…>
Branches:
master
Children:
2baee74, 6d6e012
Parents:
afb3c76 (diff), 2f526b3 (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.
Message:

merge of '03068a89c26b0986a8bf2b6f36cb478f565664eb'

and 'c3c31953c884c3aafb142e05c2dbef2809516d9c'

Files:
11 edited

Legend:

Unmodified
Added
Removed
  • apps/routerconsole/java/src/net/i2p/router/web/PluginUpdateChecker.java

    rafb3c76 r16bec08f  
    132132            String proxyHost = _context.getProperty(ConfigUpdateHandler.PROP_PROXY_HOST, ConfigUpdateHandler.DEFAULT_PROXY_HOST);
    133133            int proxyPort = _context.getProperty(ConfigUpdateHandler.PROP_PROXY_PORT, ConfigUpdateHandler.DEFAULT_PROXY_PORT_INT);
     134            _baos.reset();
    134135            try {
    135136                _get = new PartialEepGet(_context, proxyHost, proxyPort, _baos, _xpi2pURL, TrustedUpdate.HEADER_BYTES);
  • core/java/src/net/i2p/data/DataHelper.java

    rafb3c76 r16bec08f  
    10791079        in.initialize(new ByteArrayInputStream(orig, offset, length));
    10801080       
     1081        // don't make this a static field, or else I2PAppContext gets initialized too early
    10811082        ByteCache cache = ByteCache.getInstance(8, MAX_UNCOMPRESSED);
    10821083        ByteArray outBuf = cache.acquire();
  • core/java/src/net/i2p/util/ByteCache.java

    rafb3c76 r16bec08f  
    1414 * should be held onto as long as the  data referenced in it is needed.
    1515 *
     16 * Heap size control - survey of usage (April 2010) :
     17 *
     18 *  </pre>
     19        Size    Max     MaxMem  From
     20
     21        16      16      256     CryptixAESEngine
     22        16      32      512     BloomFilterIVValidator
     23        16      64      1K      UDP PacketBuilder
     24        16      128     2K      tunnel HopProcessor
     25        16      128     2K      tunnel TrivialPreprocessor
     26        16      128     2K      tunnel InboundEndpointProcessor
     27        16      128     2K      tunnel OutboundGatewayProcessor
     28
     29        32      64      2K      UDP PacketBuilder
     30        32      128     4K      tunnel TrivialPreprocessor
     31
     32        1K      32      32K     tunnel TrivialPreprocessor
     33        1K      512     512K    tunnel FragmentHandler
     34        1K      512     512K    I2NP TunnelDataMessage
     35        1K      512     512K    tunnel FragmentedMessage
     36
     37        1730    128     216K    streaming MessageOutputStream
     38
     39        2K      64      128K    UDP IMS
     40
     41        4K      32      128K    I2PTunnelRunner
     42
     43        8K      8       64K     I2PTunnel HTTPResponseOutputStream
     44
     45        32K     4       128K    SAM StreamSession
     46        32K     10      320K    SAM v2StreamSession
     47        32K     64      2M      UDP OMS
     48        32K     128     4M      streaming MessageInputStream
     49
     50        36K     64      2.25M   streaming PacketQueue
     51
     52        40K     8       320K    DataHelper decompress
     53
     54        64K     64      4M      UDP MessageReceiver - disabled in 0.7.14
     55 *  </pre>
     56 *
    1657 */
    1758public final class ByteCache {
    18     private final static Map _caches = new HashMap(16);
     59
     60    private static final Map<Integer, ByteCache> _caches = new HashMap(16);
     61
     62    /**
     63     *  max size in bytes of each cache
     64     *  Set to max memory / 128, with a min of 128KB and a max of 4MB
     65     *
     66     *  @since 0.7.14
     67     */
     68    private static final int MAX_CACHE;
     69    static {
     70        long maxMemory = Runtime.getRuntime().maxMemory();
     71        MAX_CACHE = (int) Math.min(4*1024*1024l, Math.max(128*1024l, maxMemory / 128));
     72    }
     73
    1974    /**
    2075     * Get a cache responsible for objects of the given size
     
    2277     * @param cacheSize how large we want the cache to grow before using on
    2378     *                  demand allocation
     79     *                  Since 0.7.14, a limit of 1MB / size is enforced
     80     *                  for the typical 128MB max memory JVM
    2481     * @param size how large should the objects cached be?
    2582     */
    2683    public static ByteCache getInstance(int cacheSize, int size) {
     84        if (cacheSize * size > MAX_CACHE)
     85            cacheSize = MAX_CACHE / size;
    2786        Integer sz = Integer.valueOf(size);
    2887        ByteCache cache = null;
     
    3089            if (!_caches.containsKey(sz))
    3190                _caches.put(sz, new ByteCache(cacheSize, size));
    32             cache = (ByteCache)_caches.get(sz);
     91            cache = _caches.get(sz);
    3392        }
    3493        cache.resize(cacheSize);
     94        //I2PAppContext.getGlobalContext().logManager().getLog(ByteCache.class).error("ByteCache size: " + size + " max: " + cacheSize, new Exception("from"));
    3595        return cache;
    3696    }
     97
     98    /**
     99     *  Clear everything (memory pressure)
     100     *  @since 0.7.14
     101     */
     102    public static void clearAll() {
     103        for (ByteCache bc : _caches.values())
     104            bc.clear();
     105        I2PAppContext.getGlobalContext().logManager().getLog(ByteCache.class).error("WARNING: Low memory, clearing byte caches");
     106    }
     107
    37108    private Log _log;
    38109    /** list of available and available entries */
     
    58129        SimpleScheduler.getInstance().addPeriodicEvent(new Cleanup(), CLEANUP_FREQUENCY);
    59130        _log = I2PAppContext.getGlobalContext().logManager().getLog(ByteCache.class);
     131        I2PAppContext.getGlobalContext().statManager().createRateStat("byteCache.memory." + entrySize, "Memory usage (B)", "Router", new long[] { 60*1000 });
    60132    }
    61133   
     
    64136        _maxCached = maxCachedEntries;
    65137        // make a bigger one, move the cached items over
    66         Queue newLBQ = new LinkedBlockingQueue(maxCachedEntries);
     138        Queue<ByteArray> newLBQ = new LinkedBlockingQueue(maxCachedEntries);
    67139        ByteArray ba;
    68140        while ((ba = _available.poll()) != null)
     
    110182    }
    111183   
     184    /**
     185     *  Clear everything (memory pressure)
     186     *  @since 0.7.14
     187     */
     188    private void clear() {
     189        _available.clear();
     190    }
     191
    112192    private class Cleanup implements SimpleTimer.TimedEvent {
    113193        public void timeReached() {
     194            I2PAppContext.getGlobalContext().statManager().addRateData("byteCache.memory." + _entrySize, _entrySize * _available.size(), 0);
    114195            if (System.currentTimeMillis() - _lastOverflow > EXPIRE_PERIOD) {
    115196                // we haven't exceeded the cache size in a few minutes, so lets
  • core/java/src/net/i2p/util/ResettableGZIPInputStream.java

    rafb3c76 r16bec08f  
    212212    }
    213213   
     214/******
    214215    public static void main(String args[]) {
    215216        for (int i = 129; i < 64*1024; i++) {
     
    280281        }
    281282    }
     283******/
    282284}
  • core/java/src/net/i2p/util/ResettableGZIPOutputStream.java

    rafb3c76 r16bec08f  
    123123    }
    124124   
     125/******
    125126    public static void main(String args[]) {
    126127        for (int i = 0; i < 2; i++)
     
    166167    }
    167168   
    168     /** just for testing/verification, expose the CRC32 values */
     169    // just for testing/verification, expose the CRC32 values
    169170    private static final class SnoopGZIPOutputStream extends GZIPOutputStream {
    170171        public SnoopGZIPOutputStream(OutputStream o) throws IOException {
     
    173174        public CRC32 getCRC() { return crc; }
    174175    }
     176******/
    175177}
    176178
  • core/java/src/net/i2p/util/ReusableGZIPInputStream.java

    rafb3c76 r16bec08f  
    4949    private ReusableGZIPInputStream() { super(); }
    5050   
     51/*******
    5152    public static void main(String args[]) {
    5253        for (int i = 0; i < 2; i++)
     
    128129        }
    129130    }
     131******/
    130132}
    131133
  • core/java/src/net/i2p/util/ReusableGZIPOutputStream.java

    rafb3c76 r16bec08f  
    6666    public byte[] getData() { return _buffer.toByteArray(); }
    6767   
     68/******
    6869    public static void main(String args[]) {
    6970        try {
     
    130131        }
    131132    }
     133*****/
    132134}
    133135
  • history.txt

    rafb3c76 r16bec08f  
     12010-05-02 zzz
     2    * ByteCache:
     3      - Add a per-cache stat
     4      - Limit each cache based on max memory
     5      - Disable in UDP MessageReceiver
     6      - Add clearAll() method to be called when under
     7        severe memory pressure; call from Router
     8    * Plugins: Fix version checker bug
     9
    1102010-04-27 zzz
    211    * i2psnark: Serve downloaded files from the servlet rather
  • router/java/src/net/i2p/router/Router.java

    rafb3c76 r16bec08f  
    4242import net.i2p.stat.RateStat;
    4343import net.i2p.stat.StatManager;
     44import net.i2p.util.ByteCache;
    4445import net.i2p.util.FileUtil;
    4546import net.i2p.util.I2PAppThread;
     
    225226        _oomListener = new I2PThread.OOMEventListener() {
    226227            public void outOfMemory(OutOfMemoryError oom) {
     228                ByteCache.clearAll();
    227229                _log.log(Log.CRIT, "Thread ran out of memory", oom);
    228230                for (int i = 0; i < 5; i++) { // try this 5 times, in case it OOMs
     
    253255     */
    254256    public void setKillVMOnEnd(boolean shouldDie) { _killVMOnEnd = shouldDie; }
     257
     258    /** @deprecated unused */
    255259    public boolean getKillVMOnEnd() { return _killVMOnEnd; }
    256260   
     
    924928
    925929    private void finalShutdown(int exitCode) {
    926         _log.log(Log.CRIT, "Shutdown(" + exitCode + ") complete", new Exception("Shutdown"));
     930        _log.log(Log.CRIT, "Shutdown(" + exitCode + ") complete"  /* , new Exception("Shutdown") */ );
    927931        try { _context.logManager().shutdown(); } catch (Throwable t) { }
    928932        if (ALLOW_DYNAMIC_KEYS) {
     
    13581362/* following classes are now private static inner classes, didn't bother to reindent */
    13591363
     1364private static final long LOW_MEMORY_THRESHOLD = 5 * 1024 * 1024;
     1365
    13601366/**
    13611367 * coalesce the stats framework every minute
     
    13641370private static class CoalesceStatsEvent implements SimpleTimer.TimedEvent {
    13651371    private RouterContext _ctx;
     1372    private long _maxMemory;
     1373
    13661374    public CoalesceStatsEvent(RouterContext ctx) {
    13671375        _ctx = ctx;
     
    13741382        ctx.statManager().createRateStat("router.highCapacityPeers", "How many high capacity peers we know", "Throttle", new long[] { 5*60*1000, 60*60*1000 });
    13751383        ctx.statManager().createRateStat("router.fastPeers", "How many fast peers we know", "Throttle", new long[] { 5*60*1000, 60*60*1000 });
    1376         long max = Runtime.getRuntime().maxMemory() / (1024*1024);
    1377         ctx.statManager().createRateStat("router.memoryUsed", "(Bytes) Max is " + max + "MB", "Router", new long[] { 60*1000 });
     1384        _maxMemory = Runtime.getRuntime().maxMemory();
     1385        ctx.statManager().createRateStat("router.memoryUsed", "(Bytes) Max is " + (_maxMemory / (1024*1024)) + "MB", "Router", new long[] { 60*1000 });
    13781386    }
    13791387    private RouterContext getContext() { return _ctx; }
     
    13961404        long used = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory();
    13971405        getContext().statManager().addRateData("router.memoryUsed", used, 0);
     1406        if (_maxMemory - used < LOW_MEMORY_THRESHOLD)
     1407            ByteCache.clearAll();
    13981408
    13991409        getContext().tunnelDispatcher().updateParticipatingStats(COALESCE_TIME);
  • router/java/src/net/i2p/router/RouterVersion.java

    rafb3c76 r16bec08f  
    1919    public final static String ID = "Monotone";
    2020    public final static String VERSION = CoreVersion.VERSION;
    21     public final static long BUILD = 1;
     21    public final static long BUILD = 2;
    2222
    2323    /** for example "-test" */
  • router/java/src/net/i2p/router/transport/udp/MessageReceiver.java

    rafb3c76 r16bec08f  
    1111import net.i2p.data.i2np.I2NPMessageImpl;
    1212import net.i2p.router.RouterContext;
    13 import net.i2p.util.ByteCache;
     13//import net.i2p.util.ByteCache;
    1414import net.i2p.util.I2PThread;
    1515import net.i2p.util.Log;
     
    2727    private final BlockingQueue<InboundMessageState> _completeMessages;
    2828    private boolean _alive;
    29     private ByteCache _cache;
     29    //private ByteCache _cache;
    3030    private static final int THREADS = 5;
    3131    private static final long POISON_IMS = -99999999999l;
     
    3636        _transport = transport;
    3737        _completeMessages = new LinkedBlockingQueue();
    38         _cache = ByteCache.getInstance(64, I2NPMessage.MAX_SIZE);
     38        // the runners run forever, no need to have a cache
     39        //_cache = ByteCache.getInstance(64, I2NPMessage.MAX_SIZE);
    3940        _context.statManager().createRateStat("udp.inboundExpired", "How many messages were expired before reception?", "udp", UDPTransport.RATES);
    4041        _context.statManager().createRateStat("udp.inboundRemaining", "How many messages were remaining when a message is pulled off the complete queue?", "udp", UDPTransport.RATES);
     
    9293    public void loop(I2NPMessageHandler handler) {
    9394        InboundMessageState message = null;
    94         ByteArray buf = _cache.acquire();
     95        //ByteArray buf = _cache.acquire();
     96        ByteArray buf = new ByteArray(new byte[I2NPMessage.MAX_SIZE]);
    9597        while (_alive) {
    9698            int expired = 0;
     
    143145       
    144146        // no need to zero it out, as these buffers are only used with an explicit getCompleteSize
    145         _cache.release(buf, false);
     147        //_cache.release(buf, false);
    146148    }
    147149   
Note: See TracChangeset for help on using the changeset viewer.