Changeset 7e0d0e2


Ignore:
Timestamp:
Mar 22, 2011 10:10:15 PM (9 years ago)
Author:
zzz <zzz@…>
Branches:
master
Children:
c85931c
Parents:
5dc9214
Message:
  • Implement getNames()
  • Use getNames() for merging to hosts.txt naming services to avoid O(n2)
  • Fix naming service selection
  • Don't merge from master book unless publishing
  • Add naming service and direct config options
Files:
4 edited

Legend:

Unmodified
Added
Removed
  • apps/addressbook/java/src/net/i2p/addressbook/Daemon.java

    r5dc9214 r7e0d0e2  
    2828import java.util.List;
    2929import java.util.Map;
     30import java.util.Properties;
     31import java.util.Set;
    3032
    3133import net.i2p.I2PAppContext;
     
    5557     *            The master AddressBook. This address book is never
    5658     *            overwritten, so it is safe for the user to write to.
     59     *            It is only merged to the published addressbook.
     60     *            May be null.
    5761     * @param router
    5862     *            The router AddressBook. This is the address book read by
     
    6165     *            The published AddressBook. This address book is published on
    6266     *            the user's eepsite so that others may subscribe to it.
     67     *            May be null.
    6368     *            If non-null, overwrite with the new addressbook.
    6469     * @param subscriptions
     
    6772     * @param log
    6873     *            The log to write changes and conflicts to.
     74     *            May be null.
    6975     */
    7076    public static void update(AddressBook master, AddressBook router,
    7177            File published, SubscriptionList subscriptions, Log log) {
    72         router.merge(master, true, null);
    7378        Iterator<AddressBook> iter = subscriptions.iterator();
    7479        while (iter.hasNext()) {
     
    7782        }
    7883        router.write();
    79         if (published != null)
     84        if (published != null) {
     85            if (master != null)
     86                router.merge(master, true, null);
    8087            router.write(published);
     88        }
    8189        subscriptions.write();
    8290    }
     
    8593     * Update the router and published address books using remote data from the
    8694     * subscribed address books listed in subscriptions.
     95     * Merging of the "master" addressbook is NOT supported.
    8796     *
    8897     * @param router
    89      *            The router AddressBook. This is the address book read by
     98     *            The NamingService to update, generally the root NamingService from the context.
    9099     *            client applications.
    91100     * @param published
    92101     *            The published AddressBook. This address book is published on
    93102     *            the user's eepsite so that others may subscribe to it.
     103     *            May be null.
    94104     *            If non-null, overwrite with the new addressbook.
    95105     * @param subscriptions
     
    98108     * @param log
    99109     *            The log to write changes and conflicts to.
     110     *            May be null.
    100111     * @since 0.8.6
    101112     */
    102113    public static void update(NamingService router, File published, SubscriptionList subscriptions, Log log) {
     114        // If the NamingService is a database, we look up as we go.
     115        // If it is a text file, we do things differently, to avoid O(n**2) behavior
     116        // when scanning large subscription results (i.e. those that return the whole file, not just the new entries) -
     117        // we load all the known hostnames into a Set one time.
     118        String nsClass = router.getClass().getSimpleName();
     119        boolean isTextFile = nsClass.equals("HostsTxtNamingService") || nsClass.equals("SingleFileNamingService");
     120        Set<String> knownNames = null;
     121
    103122        NamingService publishedNS = null;
    104123        Iterator<AddressBook> iter = subscriptions.iterator();
     
    115134                Map.Entry<String, String> entry = eIter.next();
    116135                String key = entry.getKey();
    117                 Destination oldDest = router.lookup(key);
     136                boolean isKnown;
     137                Destination oldDest = null;
     138                if (isTextFile) {
     139                    if (knownNames == null) {
     140                        // load the hostname set
     141                        Properties opts = new Properties();
     142                        opts.setProperty("file", "hosts.txt");
     143                        knownNames = router.getNames(opts);
     144                    }
     145                    isKnown = knownNames.contains(key);
     146                } else {
     147                    oldDest = router.lookup(key);
     148                    isKnown = oldDest != null;
     149                }
    118150                try {
    119                     if (oldDest == null) {
     151                    if (!isKnown) {
    120152                        if (AddressBook.isValidKey(key)) {
    121153                            Destination dest = new Destination(entry.getValue());
     
    136168                                    log.append("Save to published addressbook " + published.getAbsolutePath() + " failed for new key " + key);
    137169                            }
     170                            if (isTextFile)
     171                                // keep track for later dup check
     172                                knownNames.add(key);
    138173                            nnew++;
    139174                        } else if (log != null) {
     
    142177                            invalid++;
    143178                        }       
    144                     } else if (DEBUG && log != null) {
    145                         if (!oldDest.toBase64().equals(entry.getValue())) {
     179                    } else if (false && DEBUG && log != null) {
     180                        // lookup the conflict if we haven't yet (O(n**2) for text file)
     181                        if (isTextFile)
     182                            oldDest = router.lookup(key);
     183                        if (oldDest != null && !oldDest.toBase64().equals(entry.getValue())) {
    146184                            log.append("Conflict for " + key + " from "
    147185                                       + sub.getLocation()
     
    171209            sub.delete();
    172210        }
     211        subscriptions.write();
    173212    }
    174213
     
    182221     */
    183222    public static void update(Map<String, String> settings, String home) {
    184         File masterFile = new File(home, settings
    185                 .get("master_addressbook"));
    186         File routerFile = new File(home, settings
    187                 .get("router_addressbook"));
    188223        File published = null;
    189         if ("true".equals(settings.get("should_publish")))
     224        boolean should_publish = Boolean.valueOf(settings.get("should_publish")).booleanValue();
     225        if (should_publish)
    190226            published = new File(home, settings
    191227                .get("published_addressbook"));
     
    205241        }
    206242        delay *= 60 * 60 * 1000;
    207 
    208         AddressBook master = new AddressBook(masterFile);
    209         AddressBook router = new AddressBook(routerFile);
    210243       
    211244        List<String> defaultSubs = new LinkedList();
     
    218251        Log log = new Log(logFile);
    219252
    220         if (true)
    221             update(getNamingService(), published, subscriptions, log);
    222         else
     253        // If false, add hosts via naming service; if true, write hosts.txt file directly
     254        // Default false
     255        if (Boolean.valueOf(settings.get("update_direct")).booleanValue()) {
     256            // Direct hosts.txt access
     257            File routerFile = new File(home, settings.get("router_addressbook"));
     258            AddressBook master;
     259            if (should_publish) {
     260                File masterFile = new File(home, settings.get("master_addressbook"));
     261                master = new AddressBook(masterFile);
     262            } else {
     263                master = null;
     264            }
     265            AddressBook router = new AddressBook(routerFile);
    223266            update(master, router, published, subscriptions, log);
     267        } else {
     268            // Naming service - no merging of master to router and published is supported.
     269            update(getNamingService(settings.get("naming_service")), published, subscriptions, log);
     270        }
    224271    }
    225272
     
    228275    {
    229276        String name = ns.getName();
    230         if (name == srch)
    231                 return ns;
     277        if (name.equals(srch) || name.endsWith('/' + srch) || name.endsWith('\\' + srch))
     278            return ns;
    232279        List<NamingService> list = ns.getNamingServices();
    233280        if (list != null) {
     
    241288    }
    242289
    243     /** @return the NamingService for the current file name, or the root NamingService */
    244     private static NamingService getNamingService()
     290    /** @return the configured NamingService, or the root NamingService */
     291    private static NamingService getNamingService(String srch)
    245292    {
    246293        NamingService root = I2PAppContext.getGlobalContext().namingService();
    247         NamingService rv = searchNamingService(root, "hosts.txt");
     294        NamingService rv = searchNamingService(root, srch);
    248295        return rv != null ? rv : root;               
    249296    }
     
    288335        defaultSettings.put("last_fetched", "last_fetched");
    289336        defaultSettings.put("update_delay", "12");
     337        defaultSettings.put("update_direct", "false");
     338        defaultSettings.put("naming_service", "hosts.txt");
    290339       
    291340        if (!homeFile.exists()) {
  • core/java/src/net/i2p/client/naming/HostsTxtNamingService.java

    r5dc9214 r7e0d0e2  
    99
    1010import java.util.ArrayList;
     11import java.util.HashSet;
    1112import java.util.List;
    1213import java.util.Properties;
     14import java.util.Set;
    1315import java.util.StringTokenizer;
    1416
     
    7880        return super.remove(hostname.toLowerCase(), options);
    7981    }
     82
     83    /**
     84     *  All services aggregated, unless options contains
     85     *  the property "file", in which case only for that file
     86     */
     87    @Override
     88    public Set<String> getNames(Properties options) {
     89        String file = null;
     90        if (options != null)
     91            file = options.getProperty("file");
     92        if (file == null)
     93            return super.getNames(options);
     94        for (NamingService ns : _services) {
     95             String name = ns.getName();
     96             if (name.equals(file) || name.endsWith('/' + file) || name.endsWith('\\' + file))
     97                 return ns.getNames(options);
     98        }
     99        return new HashSet(0);
     100    }
    80101}
  • core/java/src/net/i2p/client/naming/MetaNamingService.java

    r5dc9214 r7e0d0e2  
    55import java.util.Collections;
    66import java.util.HashMap;
     7import java.util.HashSet;
    78import java.util.List;
    89import java.util.Map;
    910import java.util.Properties;
     11import java.util.Set;
    1012import java.util.StringTokenizer;
    1113import java.util.concurrent.CopyOnWriteArrayList;
     
    179181     */
    180182    @Override
     183    public Set<String> getNames(Properties options) {
     184        Set<String> rv = new HashSet();
     185        for (NamingService ns : _services) {
     186             rv.addAll(ns.getNames(options));
     187        }
     188        return rv;
     189    }
     190
     191    /**
     192     *  All services aggregated
     193     */
     194    @Override
    181195    public int size(Properties options) {
    182196        int rv = 0;
  • core/java/src/net/i2p/client/naming/SingleFileNamingService.java

    r5dc9214 r7e0d0e2  
    1818import java.util.Collections;
    1919import java.util.HashMap;
     20import java.util.HashSet;
    2021import java.util.Iterator;
    2122import java.util.List;
     
    366367    }
    367368
     369    /**
     370     *  @param options ignored
     371     *  @return all known host names, unsorted
     372     */
     373    public Set<String> getNames(Properties options) {
     374        if (!_file.exists())
     375            return Collections.EMPTY_SET;
     376        BufferedReader in = null;
     377        getReadLock();
     378        try {
     379            in = new BufferedReader(new InputStreamReader(new FileInputStream(_file), "UTF-8"), 16*1024);
     380            String line = null;
     381            Set<String> rv = new HashSet();
     382            while ( (line = in.readLine()) != null) {
     383                if (line.length() <= 0)
     384                    continue;
     385                if (line.startsWith("#"))
     386                    continue;
     387                int split = line.indexOf('=');
     388                if (split <= 0)
     389                    continue;
     390                String key = line.substring(0, split);
     391                rv.add(key);
     392            }
     393            return rv;
     394        } catch (IOException ioe) {
     395            _log.error("getNames error", ioe);
     396            return Collections.EMPTY_SET;
     397        } finally {
     398            if (in != null) try { in.close(); } catch (IOException ioe) {}
     399            releaseReadLock();
     400        }
     401    }
     402
    368403    /**
    369404     *  @param options ignored
Note: See TracChangeset for help on using the changeset viewer.