change the tunable for fetching history when not found in file history cache

fixes #917

Author: vladak

src/org/opensolaris/opengrok/configuration/Configuration.java

@@ -18,7 +18,7 @@
  */
 
 /*
- * Copyright (c) 2007, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2007, 2015, Oracle and/or its affiliates. All rights reserved.
  */
 package org.opensolaris.opengrok.configuration;
 
@@ -86,7 +86,7 @@ public final class Configuration {
      */
     private Project defaultProject;
     /**
-     * Default size of memory to be used for flushing of lucene docs
+     * Default size of memory to be used for flushing of Lucene docs
      * per thread.
      * Lucene 4.x uses 16MB and 8 threads, so below is a nice tunable.
      */
@@ -126,6 +126,13 @@ public final class Configuration {
     private int tabSize;
     private int command_timeout;
     private static final Logger logger = Logger.getLogger(Configuration.class.getName());
+    /*
+     * Set if we want to disable fetching history of individual files
+     * (by running appropriate SCM command) when the history is not found
+     * in history cache for repositories capable of fetching history for
+     * directories. This option affects file based history cache only.
+     */
+    private boolean noFetchHistoryWhenNoCache;
 
     public static final double defaultRamBufferSize=16;
     public static final int defaultScanningDepth=3;
@@ -168,10 +175,6 @@ public int getTabSize() {
     /**
      * Set the default tab size (number of space characters per tab character)
      * to use for each project. If {@code <= 0} tabs are read/write as is.
-     *
-     *
-     *
-
      *
      * @param tabSize tabsize to set.
      * @see Project#setTabSize(int)
@@ -239,6 +242,7 @@ public Configuration() {
         setSourceRoot(null);
         setDataRoot(null);
         setCommandTimeout(600); // 10 minutes
+        setFetchHistoryWhenNotInCache(true);
     }
 
     public String getRepoCmd(String clazzName) {
@@ -330,6 +334,14 @@ public void setHistoryCacheTime(int historyCacheTime) {
         this.historyCacheTime = historyCacheTime;
     }
 
+    public boolean isFetchHistoryWhenNotInCache() {
+        return noFetchHistoryWhenNoCache;
+    }
+
+    public void setFetchHistoryWhenNotInCache(boolean nofetch) {
+        this.noFetchHistoryWhenNoCache = nofetch;
+    }
+
     /**
      * Should the history cache be stored in a database? If yes,
      * {@code JDBCHistoryCache} will be used to cache the history; otherwise,

src/org/opensolaris/opengrok/configuration/RuntimeEnvironment.java

@@ -18,7 +18,7 @@
  */
 
 /*
- * Copyright (c) 2006, 2014, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2006, 2015, Oracle and/or its affiliates. All rights reserved.
  */
 package org.opensolaris.opengrok.configuration;
 
@@ -887,16 +887,14 @@ public void setChattyStatusPage(boolean chatty) {
         threadConfig.get().setChattyStatusPage(chatty);
     }
 
-    public boolean noFetchHistoryWhenNotInCache() {
-        String enabled =
-            System.getProperty("org.opensolaris.opengrok.history.noFetchWhenNotInCache");
-        if (enabled != null) {
-            return true;
-        }
+    public void setFetchHistoryWhenNotInCache(boolean nofetch) {
+        threadConfig.get().setFetchHistoryWhenNotInCache(nofetch);
+    }
 
-        return (false);
+    public boolean isFetchHistoryWhenNotInCache() {
+        return threadConfig.get().isFetchHistoryWhenNotInCache();
     }
-    
+
     /**
      * Read an configuration file and set it as the current configuration.
      *

src/org/opensolaris/opengrok/history/FileHistoryCache.java

@@ -18,7 +18,7 @@
  */
 
 /*
- * Copyright (c) 2008, 2014, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2015, Oracle and/or its affiliates. All rights reserved.
  */
 
 package org.opensolaris.opengrok.history;
@@ -476,11 +476,17 @@ public History get(File file, Repository repository, boolean withFiles)
             }
         }
 
-        // Some repository checkouts may contain lots of files untracked by
-        // given SCM. For these it would be waste of time to get their history.
+        /*
+         * Some mirrors of repositories which are capable of fetching history
+         * for directories may contain lots of files untracked by given SCM.
+         * For these it would be waste of time to get their history
+         * since the history of all files in this repository should have been
+         * fetched in the first phase of indexing.
+         */
         RuntimeEnvironment env = RuntimeEnvironment.getInstance();
-        if (isHistoryIndexDone() && env.noFetchHistoryWhenNotInCache()) {
-            return null;
+        if (isHistoryIndexDone() && repository.hasHistoryForDirectories() &&
+            !env.isFetchHistoryWhenNotInCache()) {
+                return null;
         }
 
         final History history;

test/org/opensolaris/opengrok/configuration/RuntimeEnvironmentTest.java

@@ -18,7 +18,7 @@
  */
 
 /*
- * Copyright (c) 2008, 2011, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2015, Oracle and/or its affiliates. All rights reserved.
  */
 package org.opensolaris.opengrok.configuration;
 
@@ -171,6 +171,14 @@ public void testHistoryReaderTimeLimit() {
         assertEquals(50, instance.getHistoryReaderTimeLimit());
     }
 
+    @Test
+    public void testFetchHistoryWhenNotInCache() {
+        RuntimeEnvironment instance = RuntimeEnvironment.getInstance();
+        assertEquals(true, instance.isFetchHistoryWhenNotInCache());
+        instance.setFetchHistoryWhenNotInCache(false);
+        assertEquals(false, instance.isFetchHistoryWhenNotInCache());
+    }
+
     @Test
     public void testUseHistoryCache() {
         RuntimeEnvironment instance = RuntimeEnvironment.getInstance();

test/org/opensolaris/opengrok/history/FileHistoryCacheTest.java

@@ -383,4 +383,50 @@ public void testRenamedFile() throws Exception {
         updatedHistory = cache.get(reposRoot, repo, true);
         assertEquals(14, updatedHistory.getHistoryEntries().size());
     }
+
+    private void checkNoHistoryFetchRepo(String reponame, String filename,
+        boolean hasHistory, boolean historyFileExists) throws Exception {
+
+        File reposRoot = new File(repositories.getSourceRoot(), reponame);
+        Repository repo = RepositoryFactory.getRepository(reposRoot);
+
+        // Make sure the file exists in the repository.
+        File repoFile = new File(reposRoot, filename);
+        assertTrue(repoFile.exists());
+
+        // Try to fetch the history for given file. With default setting of
+        // FetchHistoryWhenNotInCache this should create corresponding file
+        // in history cache.
+        History retrievedHistory = cache.get(repoFile, repo, true);
+        assertEquals(hasHistory, retrievedHistory == null ? false : true);
+
+        // The file in history cache should not exist since
+        // FetchHistoryWhenNotInCache is set to false.
+        File dataRoot = new File(repositories.getDataRoot(),
+            "historycache" + File.separatorChar + reponame);
+        File fileHistory = new File(dataRoot, filename + ".gz");
+        assertEquals(historyFileExists, fileHistory.exists());
+    }
+
+    /*
+     * Functional test for the FetchHistoryWhenNotInCache configuration option.
+     */
+    public void testNoHistoryFetch() throws Exception {
+        // Do not create history cache for files which do not have it cached.
+        RuntimeEnvironment.getInstance().setFetchHistoryWhenNotInCache(false);
+
+        // Make cache.get() predictable. Normally when the retrieval of
+        // history of given file is faster than the limit, the history of this
+        // file is not stored. For the sake of this test we want the history
+        // to be always stored.
+        RuntimeEnvironment.getInstance().setHistoryReaderTimeLimit(0);
+
+        // Pretend we are done with first phase of indexing.
+        cache.setHistoryIndexDone();
+
+        // First try repo with ability to fetch history for directories.
+        checkNoHistoryFetchRepo("mercurial", "main.c", false, false);
+        // Second try repo which can fetch history of individual files only.
+        checkNoHistoryFetchRepo("teamware", "header.h", true, true);
+    }
 }