Changeset 4322cb3

Timestamp:

Dec 17, 2011 6:52:23 PM (9 years ago)

Author:

zzz <zzz@…>

Branches:

master

Children:

a2454e8e

Parents:

a7311a57 (diff), b18e7c7 (diff)
Note: this is a merge changeset, the changes displayed below correspond to the merge itself.
Use the (diff) links above to see all the changes relative to each parent.

Message:

merge of '0035c4f64c2b0f23c9e007363b634f360e2f10cb'

and '0c2d5fea1f036773ef51ce22d9d8c4e9203ee5f3'

Files:

• apps/i2psnark/java/src/org/klomp/snark/Storage.java (modified) (3 diffs)
• apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java (modified) (1 diff)
• build.xml (modified) (1 diff)
• core/java/src/net/i2p/util/SimpleTimer2.java (modified) (1 diff)
• history.txt (modified) (1 diff)
• router/java/src/net/i2p/router/InNetMessagePool.java (modified) (1 diff)
• router/java/src/net/i2p/router/RouterVersion.java (modified) (1 diff)
• router/java/src/net/i2p/router/message/GarlicMessageReceiver.java (modified) (1 diff)
• router/java/src/net/i2p/router/peermanager/ProfileManagerImpl.java (modified) (26 diffs)
• router/java/src/net/i2p/router/peermanager/ProfileOrganizer.java (modified) (5 diffs)
• router/java/src/net/i2p/router/transport/GeoIP.java (modified) (1 diff)
• router/java/src/net/i2p/router/transport/udp/IntroductionManager.java (modified) (1 diff)
• router/java/src/net/i2p/router/tunnel/HopProcessor.java (modified) (3 diffs)
• router/java/src/net/i2p/router/tunnel/InboundEndpointProcessor.java (modified) (1 diff)
• router/java/src/net/i2p/router/tunnel/pool/TunnelPool.java (modified) (1 diff)

apps/i2psnark/java/src/org/klomp/snark/Storage.java

@@ -24 +24 @@
 import java.io.IOException;
 import java.io.RandomAccessFile;
+import java.nio.charset.Charset;
+import java.nio.charset.CharsetEncoder;
 import java.security.MessageDigest;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
 import java.util.StringTokenizer;
+import java.util.concurrent.ConcurrentHashMap;
 
 import net.i2p.crypto.SHA1;
@@ -67 +71 @@
   public static final int MAX_PIECES = 10*1024;
   public static final long MAX_TOTAL_SIZE = MAX_PIECE_SIZE * (long) MAX_PIECES;
+
+  private static final Map<String, String> _filterNameCache = new ConcurrentHashMap();
 
   /**
@@ -569 +575 @@
    * Removes 'suspicious' characters from the given file name.
    * http://msdn.microsoft.com/en-us/library/aa365247%28VS.85%29.aspx
+   * Then replace chars not supported in the charset.
+   *
+   * This is called frequently and it can be pretty slow so cache the result.
+   *
+   * TODO: If multiple files in the same torrent map to the same filter name,
+   * the whole torrent will blow up. Check at torrent creation?
    */
   public static String filterName(String name)
   {
-    if (name.equals(".") || name.equals(" "))
-        return "_";
-    String rv = name;
-    if (rv.startsWith("."))
-        rv = '_' + rv.substring(1);
-    if (rv.endsWith(".") || rv.endsWith(" "))
-        rv = rv.substring(0, rv.length() - 1) + '_';
-    for (int i = 0; i < ILLEGAL.length; i++) {
-        if (rv.indexOf(ILLEGAL[i]) >= 0)
-            rv = rv.replace(ILLEGAL[i], '_');
-    }
+    String rv = _filterNameCache.get(name);
+    if (rv != null)
+        return rv;
+    if (name.equals(".") || name.equals(" ")) {
+        rv = "_";
+    } else {
+        rv = name;
+        if (rv.startsWith("."))
+            rv = '_' + rv.substring(1);
+        if (rv.endsWith(".") || rv.endsWith(" "))
+            rv = rv.substring(0, rv.length() - 1) + '_';
+        for (int i = 0; i < ILLEGAL.length; i++) {
+            if (rv.indexOf(ILLEGAL[i]) >= 0)
+                rv = rv.replace(ILLEGAL[i], '_');
+        }
+        // Replace characters not supported in the charset
+        if (!Charset.defaultCharset().name().equals("UTF-8")) {
+            try {
+                CharsetEncoder enc = Charset.defaultCharset().newEncoder();
+                if (!enc.canEncode(rv)) {
+                    String repl = rv;
+                    for (int i = 0; i < rv.length(); i++) {
+                        char c = rv.charAt(i);
+                        if (!enc.canEncode(c))
+                            repl = repl.replace(c, '_');
+                    }
+                    rv = repl;
+                }
+            } catch (Exception ex) {
+                ex.printStackTrace();
+            }
+    }   }
+    _filterNameCache.put(name, rv);
     return rv;
   }

apps/i2psnark/java/src/org/klomp/snark/web/I2PSnarkServlet.java

@@ -1760 +1760 @@
             //buf.append("<br>").append(_("Maggot link")).append(": <a href=\"").append(MAGGOT).append(hex).append(':').append(hex).append("\">")
             //   .append(MAGGOT).append(hex).append(':').append(hex).append("</a>");
+            buf.append("<br>").append(_("Torrent file")).append(": ").append(snark.getName());
             buf.append("</div></th></tr>");
         }

build.xml

@@ -889 +889 @@
         </exec>
         <exec executable="echo" osfamily="unix" failifexecutionfails="true" output="pkg-temp/history.txt" append="true">
-            <arg value="EARLIER HISTORY IS AVAILABLE IN THE SOURCE PACKAGE" />
+            <arg value="&#10;&#10;----------------&#10;&#10;EARLIER HISTORY IS AVAILABLE IN THE SOURCE PACKAGE" />
         </exec>
         <copy file="installer/resources/deletelist.txt" todir="pkg-temp/" />

core/java/src/net/i2p/util/SimpleTimer2.java

@@ -236 +236 @@
                 _log.warn(_pool + " wtf, early execution " + delay + ": " + this);
             else if (_log.shouldLog(Log.WARN) && delay < -1000)
-                _log.warn(" wtf, late execution " + delay + ": " + this + _pool.debug());
+                _log.warn(" wtf, late execution " + (0 - delay) + ": " + this + _pool.debug());
             try {
                 timeReached();

history.txt

@@ +1 @@
+2011-12-17 zzz
+  * i2psnark:
+    - Replace file name characters not supported in default charset
+    - Add torrent file name to local details page
+  * GeoIP: Reduce thread priority during lookup
+  * ProfileManager: Make some methods non-blocking to reduce
+                    lock contention in transports
+
 2011-12-15 kytv
   * Swedish translation updates from Transifex

router/java/src/net/i2p/router/InNetMessagePool.java

@@ -147 +147 @@
             //    level = Log.INFO;
             if (_log.shouldLog(level))
-                _log.log(level, "Duplicate message received [" + messageBody.getUniqueId() 
-                          + " expiring on " + exp + "]: " + messageBody.getClass().getName() + ": " + invalidReason 
+                _log.log(level, "Dropping message [" + messageBody.getUniqueId() 
+                          + " expiring on " + exp + "]: " + messageBody.getClass().getSimpleName() + ": " + invalidReason 
                           + ": " + messageBody);
             _context.statManager().addRateData("inNetPool.dropped", 1, 0);
+            // FIXME not necessarily a duplicate, could be expired too long ago / too far in future
             _context.statManager().addRateData("inNetPool.duplicate", 1, 0);
             _context.messageHistory().droppedOtherMessage(messageBody, (fromRouter != null ? fromRouter.calculateHash() : fromRouterHash));
             _context.messageHistory().messageProcessingError(messageBody.getUniqueId(), 
-                                                                messageBody.getClass().getName(), 
+                                                                messageBody.getClass().getSimpleName(), 
                                                                 "Duplicate/expired");
             return -1;

router/java/src/net/i2p/router/RouterVersion.java

@@ -19 +19 @@
     public final static String ID = "Monotone";
     public final static String VERSION = CoreVersion.VERSION;
-    public final static long BUILD = 21;
+    public final static long BUILD = 22;
 
     /** for example "-test" */
-    public final static String EXTRA = "";
+    public final static String EXTRA = "-rc";
     public final static String FULL_VERSION = VERSION + "-" + BUILD + EXTRA;
     public static void main(String args[]) {

router/java/src/net/i2p/router/message/GarlicMessageReceiver.java

@@ -113 +113 @@
                            + " expiration " + howLongAgo + " ago: " + invalidReason + ": " + clove);
             _context.messageHistory().messageProcessingError(clove.getCloveId(), 
-                                                             clove.getData().getClass().getName(), 
+                                                             clove.getData().getClass().getSimpleName(), 
                                                              "Clove is not valid (expiration " + howLongAgo + " ago)");
         }

router/java/src/net/i2p/router/peermanager/ProfileManagerImpl.java

@@ -22 +22 @@
 import net.i2p.util.Log;
 
+/**
+ *  Methods to update profiles.
+ *  Unless otherwise noted, methods are blocking on the reorganize lock.
+ */
 public class ProfileManagerImpl implements ProfileManager {
     private final Log _log;
@@ -34 +38 @@
      * Note that it took msToSend to send a message of size bytesSent to the peer over the transport.
      * This should only be called if the transport considered the send successful.
-     *
+     * Non-blocking. Will not update the profile if we can't get the lock.
      */
     public void messageSent(Hash peer, String transport, long msToSend, long bytesSent) {
-        PeerProfile data = getProfile(peer);
+        PeerProfile data = getProfileNonblocking(peer);
         if (data == null) return;
         data.setLastSendSuccessful(_context.clock().now());
@@ -44 +48 @@
     
     /**
-     * Note that the router failed to send a message to the peer over the transport specified
-     *
+     * Note that the router failed to send a message to the peer over the transport specified.
+     * Non-blocking. Will not update the profile if we can't get the lock.
      */
     public void messageFailed(Hash peer, String transport) {
-        PeerProfile data = getProfile(peer);
+        PeerProfile data = getProfileNonblocking(peer);
         if (data == null) return;
         data.setLastSendFailed(_context.clock().now());
@@ -54 +58 @@
     
     /**
-     * Note that the router failed to send a message to the peer over any transport
-     *
+     * Note that the router failed to send a message to the peer over any transport.
+     * Non-blocking. Will not update the profile if we can't get the lock.
      */
     public void messageFailed(Hash peer) {
-        PeerProfile data = getProfile(peer);
+        PeerProfile data = getProfileNonblocking(peer);
         if (data == null) return;
         data.setLastSendFailed(_context.clock().now());
@@ -71 +75 @@
             _log.info("Comm error occurred for peer " + peer.toBase64(), new Exception("Comm error"));
         PeerProfile data = getProfile(peer);
-        if (data == null) return;
+        //if (data == null) return;
         data.setLastSendFailed(_context.clock().now());
     }
@@ -81 +85 @@
     public void tunnelJoined(Hash peer, long responseTimeMs) {
         PeerProfile data = getProfile(peer);
-        if (data == null) return;
+        //if (data == null) return;
         data.getTunnelCreateResponseTime().addData(responseTimeMs, responseTimeMs);
         data.setLastHeardFrom(_context.clock().now());
@@ -96 +100 @@
     public void tunnelRejected(Hash peer, long responseTimeMs, int severity) {
         PeerProfile data = getProfile(peer);
-        if (data == null) return;
+        //if (data == null) return;
         data.setLastHeardFrom(_context.clock().now());
         data.getTunnelHistory().incrementRejected(severity);
@@ -109 +113 @@
     public void tunnelTimedOut(Hash peer) {
         PeerProfile data = getProfile(peer);
-        if (data == null) return;
+        //if (data == null) return;
         data.getTunnelHistory().incrementRejected(TunnelHistory.TUNNEL_REJECT_BANDWIDTH);
     }
@@ -120 +124 @@
     public void tunnelTestSucceeded(Hash peer, long responseTimeMs) {
         PeerProfile data = getProfile(peer);
-        if (data == null) return;
+        //if (data == null) return;
         data.updateTunnelTestTimeAverage(responseTimeMs);
         data.getTunnelTestResponseTime().addData(responseTimeMs, responseTimeMs);
@@ -129 +133 @@
             return;
         PeerProfile data = getProfile(peer);
-        if (data != null)
+        //if (data != null)
             data.dataPushed(size); // ignore rtt, as we are averaging over a minute
     }
+
     public void tunnelDataPushed1m(Hash peer, int size) {
         if (_context.routerHash().equals(peer))
             return;
         PeerProfile data = getProfile(peer);
-        if (data != null)
+        //if (data != null)
             data.dataPushed1m(size);
     }
@@ -145 +150 @@
             return;
         PeerProfile data = getProfile(peer);
-        if (data != null)
+        //if (data != null)
             data.tunnelDataTransferred(size);
     }
@@ -163 +168 @@
     public void tunnelFailed(Hash peer, int pct) {
         PeerProfile data = getProfile(peer);
-        if (data == null) return;
+        //if (data == null) return;
         data.setLastHeardFrom(_context.clock().now());
         data.getTunnelHistory().incrementFailed(pct);
@@ -175 +180 @@
     public void dbLookupSuccessful(Hash peer, long responseTimeMs) {
         PeerProfile data = getProfile(peer);
-        if (data == null) return;
+        //if (data == null) return;
         data.setLastHeardFrom(_context.clock().now());
         if (!data.getIsExpandedDB())
@@ -192 +197 @@
     public void dbLookupFailed(Hash peer) {
         PeerProfile data = getProfile(peer);
-        if (data == null) return;
+        //if (data == null) return;
         if (!data.getIsExpandedDB())
             data.expandDBProfile();
@@ -209 +214 @@
     public void dbLookupReply(Hash peer, int newPeers, int oldPeers, int invalid, int duplicate, long responseTimeMs) {
         PeerProfile data = getProfile(peer);
-        if (data == null) return;
+        //if (data == null) return;
         data.setLastHeardFrom(_context.clock().now());
         if (!data.getIsExpandedDB())
@@ -225 +230 @@
     public void dbLookupReceived(Hash peer) {
         PeerProfile data = getProfile(peer);
-        if (data == null) return;
+        //if (data == null) return;
         data.setLastHeardFrom(_context.clock().now());
         if (!data.getIsExpandedDB())
@@ -239 +244 @@
     public void dbStoreReceived(Hash peer, boolean wasNewKey) {
         PeerProfile data = getProfile(peer);
-        if (data == null) return;
+        //if (data == null) return;
         data.setLastHeardFrom(_context.clock().now());
         if (!data.getIsExpandedDB())
@@ -258 +263 @@
     public void dbStoreSent(Hash peer, long responseTimeMs) {
         PeerProfile data = getProfile(peer);
-        if (data == null) return;
+        //if (data == null) return;
         long now = _context.clock().now();
         data.setLastHeardFrom(now);
@@ -276 +281 @@
     public void dbStoreSuccessful(Hash peer) {
         PeerProfile data = getProfile(peer);
-        if (data == null) return;
+        //if (data == null) return;
         long now = _context.clock().now();
         data.setLastHeardFrom(now);
@@ -294 +299 @@
     public void dbStoreFailed(Hash peer) {
         PeerProfile data = getProfile(peer);
-        if (data == null) return;
+        //if (data == null) return;
         if (!data.getIsExpandedDB())
             data.expandDBProfile();
@@ -309 +314 @@
     public void heardAbout(Hash peer) {
         PeerProfile data = getProfile(peer);
-        if (data == null) return;
+        //if (data == null) return;
         data.setLastHeardAbout(_context.clock().now());
     }
@@ -319 +324 @@
     public void heardAbout(Hash peer, long when) {
         PeerProfile data = getProfile(peer);
-        if (data == null) return;
+        //if (data == null) return;
         if (when > data.getLastHeardAbout())
             data.setLastHeardAbout(when);
@@ -328 +333 @@
      * transport.  Messages received without any "from" information aren't recorded
      * through this metric.  If msToReceive is negative, there was no timing information
-     * available
-     *
+     * available.
+     * Non-blocking. Will not update the profile if we can't get the lock.
      */
     public void messageReceived(Hash peer, String style, long msToReceive, int bytesRead) {
-        PeerProfile data = getProfile(peer);
+        PeerProfile data = getProfileNonblocking(peer);
         if (data == null) return;
         data.setLastHeardFrom(_context.clock().now());
@@ -338 +343 @@
     }
     
+    /**
+     *   Blocking.
+     *   Creates a new profile if it didn't exist.
+     *   @return non-null
+     */
     private PeerProfile getProfile(Hash peer) {
         PeerProfile prof = _context.profileOrganizer().getProfile(peer);
@@ -348 +358 @@
     }
     
-    
-    /** provide a simple summary of a number of peers, suitable for publication in the netDb */
+    /**
+     *  Non-blocking.
+     *  @return null if the profile doesn't exist, or the fetch would have blocked
+     *  @since 0.8.12
+     */
+    private PeerProfile getProfileNonblocking(Hash peer) {
+        return _context.profileOrganizer().getProfileNonblocking(peer);
+    }
+    
+    /**
+     *  provide a simple summary of a number of peers, suitable for publication in the netDb
+     *  @deprecated unused
+     */
     public Properties summarizePeers(int numPeers) {
+/****
         Set peers = new HashSet(numPeers);
         // lets get the fastest ones we've got (this fails over to include just plain reliable,
         // or even notFailing peers if there aren't enough fast ones)
         _context.profileOrganizer().selectFastPeers(numPeers, null, peers);
+****/
         Properties props = new Properties();
+/****
         for (Iterator iter  = peers.iterator(); iter.hasNext(); ) {
             Hash peer = (Hash)iter.next();
@@ -385 +409 @@
             props.setProperty("profile." + peer.toBase64().replace('=', '_'), buf.toString());
         }
+****/
         return props;
     }
     
+/****
     private final static DecimalFormat _fmt = new DecimalFormat("##0.00", new DecimalFormatSymbols(Locale.UK));
     private final static String num(double val) {
         synchronized (_fmt) { return _fmt.format(val); }
     }
+****/
 }

router/java/src/net/i2p/router/peermanager/ProfileOrganizer.java

@@ -53 +53 @@
     /** H(routerIdnetity), containing elements in _notFailingPeers */
     private final List<Hash> _notFailingPeersList;
-    /** H(routerIdentity) to PeerProfile for all peers that ARE failing horribly (but that we haven't dropped reference to yet) */
+    /** TO BE REMOVED H(routerIdentity) to PeerProfile for all peers that ARE failing horribly (but that we haven't dropped reference to yet) */
     private final Map<Hash, PeerProfile> _failingPeers;
     /** who are we? */
@@ -121 +121 @@
     }
 
+    /**
+     *  Get the lock if we can. Non-blocking.
+     *  @return true if the lock was acquired
+     *  @since 0.8.12
+     */
+    private boolean tryReadLock() {
+        return _reorganizeLock.readLock().tryLock();
+    }
+
     private void releaseReadLock() {
         _reorganizeLock.readLock().unlock();
@@ -148 +157 @@
     
     /**
-     * Retrieve the profile for the given peer, if one exists (else null)
-     *
+     * Retrieve the profile for the given peer, if one exists (else null).
+     * Blocking if a reorganize is happening.
      */
     public PeerProfile getProfile(Hash peer) {
@@ -156 +165 @@
             return locked_getProfile(peer);
         } finally { releaseReadLock(); }
+    }
+    
+    /**
+     * Retrieve the profile for the given peer, if one exists (else null).
+     * Non-blocking. Returns null if a reorganize is happening.
+     * @since 0.8.12
+     */
+    public PeerProfile getProfileNonblocking(Hash peer) {
+        if (tryReadLock()) {
+            try {
+                return locked_getProfile(peer);
+            } finally { releaseReadLock(); }
+        }
+        return null;
     }
     
@@ -244 +267 @@
     public boolean isHighCapacity(Hash peer) { return isX(_highCapacityPeers, peer); }
     public boolean isWellIntegrated(Hash peer) { return isX(_wellIntegratedPeers, peer); }
-    public boolean isFailing(Hash peer) { return isX(_failingPeers, peer); }
+
+    /**
+     *  Deprecated for now, always false
+     */
+    public boolean isFailing(Hash peer) {
+        // Always false so skip the lock
+        //return isX(_failingPeers, peer);
+        return false;
+    }
         
     /** @since 0.8.8 */

router/java/src/net/i2p/router/transport/GeoIP.java

@@ -99 +99 @@
             return;
         }
-        LookupJob j = new LookupJob();
-        j.run();
-        updateOurCountry();
+        int pri = Thread.currentThread().getPriority();
+        if (pri > Thread.MIN_PRIORITY)
+            Thread.currentThread().setPriority(pri - 1);
+        try {
+            LookupJob j = new LookupJob();
+            j.run();
+            updateOurCountry();
+        } finally {
+            if (pri > Thread.MIN_PRIORITY)
+                Thread.currentThread().setPriority(pri);
+        }
     }
 

router/java/src/net/i2p/router/transport/udp/IntroductionManager.java

@@ -127 +127 @@
                 continue;
             }
-            if (_context.profileOrganizer().isFailing(cur.getRemotePeer()) ||
+            if ( /* _context.profileOrganizer().isFailing(cur.getRemotePeer()) || */
                 _context.shitlist().isShitlisted(cur.getRemotePeer()) ||
                 _transport.wasUnreachable(cur.getRemotePeer())) {

router/java/src/net/i2p/router/tunnel/HopProcessor.java

@@ -13 +13 @@
  * inbound and outbound participants, outbound endpoints,
  * and inbound gateways (with a small modification per 
- * InbuondGatewayProcessor).  
+ * InboundGatewayProcessor).  
  *
  */
@@ -78 +78 @@
         if (!okIV) {
             if (_log.shouldLog(Log.WARN)) 
-                _log.warn("Invalid IV received on tunnel " + _config.getReceiveTunnel());
+                _log.warn("Invalid IV, dropping at hop " + _config);
             return false;
         }
@@ -112 +112 @@
         _context.aes().encryptBlock(orig, offset, _config.getIVKey(), orig, offset);
     }
+
+    /**
+     *  @since 0.8.12
+     */
+    @Override
+    public String toString() {
+        return getClass().getSimpleName() + " for " + _config;
+    }
 }

router/java/src/net/i2p/router/tunnel/InboundEndpointProcessor.java

@@ -64 +64 @@
         if (!ok) {
             if (_log.shouldLog(Log.WARN)) 
-                _log.warn("Invalid IV received");
+                _log.warn("Invalid IV, dropping at IBEP " + _config);
             _cache.release(ba);
             return false;

router/java/src/net/i2p/router/tunnel/pool/TunnelPool.java

@@ -81 +81 @@
             _inProgress.clear();
         }
-        if (_log.shouldLog(Log.WARN))
-            _log.warn(toString() + ": Startup() called, was already alive? " + _alive, new Exception());
+        if (_log.shouldLog(Log.INFO))
+            _log.info(toString() + ": Startup() called, was already alive? " + _alive, new Exception());
         _alive = true;
         _started = System.currentTimeMillis();