001    package org.hackystat.utilities.uricache;
002    
003    import java.io.File;
004    import java.io.Serializable;
005    import java.util.ArrayList;
006    import java.util.HashSet;
007    import java.util.List;
008    import java.util.Properties;
009    import java.util.Set;
010    import java.util.logging.Level;
011    import java.util.logging.Logger;
012    
013    import org.apache.jcs.access.exception.CacheException;
014    import org.apache.jcs.engine.ElementAttributes;
015    import org.apache.jcs.engine.control.CompositeCacheManager;
016    import org.apache.jcs.JCS;
017    import org.hackystat.utilities.home.HackystatUserHome;
018    import org.hackystat.utilities.logger.HackystatLogger;
019    
020    /**
021     * Provides a wrapper around Apache JCS (Java Caching System) to facilitate Hackystat caching. This
022     * wrapper provides the following:
023     * <ul>
024     * <li> Automatic configuration of an indexed disk cache backing store.
025     * <li> Write-through caching: All cached instances are written out to disk. 
026     * <li> Provides a default maximum life for expiring of entries of one day.
027     * <li> Provides a default maximum cache size of 10000 instances.
028     * <li> Provides a default directory location (inside ~/.hackystat) for backing store files. 
029     * <li> Helps ensure that all UriCache instances have a unique name.
030     * <li> All caches use the JCS "group" facility to allow access to the set of keys. 
031     * <li> Constructor uses "days" rather than seconds as time unit for maxLife.
032     * <li> put() uses "hours" rather than seconds as time unit for maxLife.
033     * <li> A more convenient API for setting/getting items from the cache and controlling logging.
034     * <li> Logging of exceptions raised by JCS.
035     * <li> Disables JCS logging messages unless the System property
036     * org.hackystat.utilities.uricache.enableJCSLogging is set.
037     * <li> Shutdown hook ensures that backing index file is closed correctly on JVM exit. 
038     * <li> Convenient packaging mechanism for required jar files to simplify library use.
039     * </ul>
040     * 
041     * Here's an example usage, where we create a separate cache for each user to hold their sensor data
042     * instances as part of the dailyprojectdata service.
043     * 
044     * <pre>
045     * SensorBaseClient client = new SensorBaseClient(user, host);
046     * UriCache cache = new UriCache(user.getEmail(), &quot;dailyprojectdata&quot;);
047     *   :
048     * SensorData data = (SensorData)cache.get(uriString);
049     * if (data == null) {
050     *   // Cache doesn't have it, so retrieve from SensorBase and cache locally for next time.
051     *   data = client.getSensorData(uriString);
052     *   cache.put(uriString, data);
053     * }
054     * </pre>
055     * 
056     * The cache files are in the directory ~/.hackystat/dailyprojectdata/uricache. Instances expire
057     * from the cache after one day, by default. The maximum number of in-memory instances is 10,000, by
058     * default.
059     * 
060     * @author Philip Johnson
061     */
062    public class UriCache {
063      
064      /** The number of seconds in a day. * */
065      private static final long secondsInADay = 60L * 60L * 24L;
066      /** Maximum life in seconds that an entry stays in the cache. Default is 1 day. */
067      private static final Long defaultMaxLifeSeconds = secondsInADay;
068      /** Maximum number of in-memory instances before sending items to disk. Default is 50,000. */
069      private static final Long defaultCapacity = 10000L;
070      /** The name of this cache, which defines a "region" in JCS terms. */
071      private String cacheName = null;
072      /** The logger used for cache exception logging. */
073      private Logger logger = null;
074      /** Holds a list of already defined caches to help ensure uniqueness. */
075      private static List<String> cacheNames = new ArrayList<String>();
076      /** Default group name. No client should ever using the following string for a group. */
077      private static final String DEFAULT_GROUP = "__Default_UriCache_Group__";
078      
079      private static final String failureMsg = "Failure to clear cache ";
080      
081      /** A thread that will ensure that all of these caches will be disposed of during shutdown. */ 
082      private static Thread shutdownThread = new Thread() {
083        /** Run the shutdown hook for disposing of all caches. */
084        @Override 
085        public void run() {
086          for (String cacheName : cacheNames) {
087            try {
088              System.out.println("Shutting down " + cacheName + " cache.");
089              JCS.getInstance(cacheName).dispose();
090            }
091            catch (Exception e) {
092              String msg = failureMsg + cacheName + ":" + e.getMessage();
093              System.out.println(msg);
094            }
095          }
096        }
097      };
098      
099      /** A boolean that enables us to figure out whether to install the shutdown thread.  */
100      private static boolean hasShutdownHook = false;  //NOPMD
101      
102      /**
103       * Creates a new UriCache instance with the specified name. Good for services who want to create a
104       * single cache for themselves, such as "dailyprojectdata". If the cache with this name has been 
105       * previously created, then this instance will become an alias to the previously existing cache. 
106       * 
107       * @param cacheName The name of this cache.
108       * @param subDir the .hackystat subdirectory in which the uricache directory holding the backing
109       *        store will be created.
110       */
111      public UriCache(String cacheName, String subDir) {
112        this(cacheName, subDir, new Double(defaultMaxLifeSeconds), defaultCapacity);
113      }
114      
115      /**
116       * Creates a new UriCache with the specified parameters. 
117       * If a cache with this name already exists, then this instance will be an alias to that cache
118       * and its original configuration will remain unchanged. 
119       * 
120       * @param cacheName The name of this UriCache, which will be used as the JCS "region" and also
121       *        define the subdirectory in which the index files will live.
122       * @param subDir the .hackystat subdirectory in which the uricache directory holding the backing
123       *        store will be created.
124       * @param maxLifeDays The maximum number of days after which items expire from the cache.
125       * @param capacity The maximum number of instances to hold in the cache. 
126       */
127      public UriCache(String cacheName, String subDir, Double maxLifeDays, Long capacity) {
128        // Set up the shutdown hook if we're the first one. Not thread safe, but there's not too
129        // much harm done if there are multiple shutdown hooks running.
130        if (!UriCache.hasShutdownHook) {
131          Runtime.getRuntime().addShutdownHook(UriCache.shutdownThread);
132          UriCache.hasShutdownHook = true;
133        }
134        this.cacheName = cacheName;
135        this.logger = HackystatLogger.getLogger(cacheName + ".uricache", subDir);
136    
137        // Finish configuration if this is a new instance of the cache.
138        if (!UriCache.cacheNames.contains(cacheName)) {
139          UriCache.cacheNames.add(cacheName);
140          if (!System.getProperties().containsKey(
141              "org.hackystat.utilities.uricache.enableJCSLogging")) {
142            Logger.getLogger("org.apache.jcs").setLevel(Level.OFF);
143          }
144          CompositeCacheManager ccm = CompositeCacheManager.getUnconfiguredInstance();
145          long maxLifeSeconds = (long) (maxLifeDays * secondsInADay);
146          ccm.configure(initJcsProps(cacheName, subDir, maxLifeSeconds, capacity));
147        }
148      }
149      
150      /**
151       * Adds the key-value pair to this cache. Entry will expire from cache after the default maxLife
152       * (currently 24 hours). Logs a message if the cache throws an exception.
153       * 
154       * @param key The key, typically a UriString.
155       * @param value The value, typically the object returned from the Hackystat service.
156       */
157      public void put(Serializable key, Serializable value) {
158        try {
159          JCS.getInstance(this.cacheName).putInGroup(key, DEFAULT_GROUP, value);
160        }
161        catch (CacheException e) {
162          String msg = "Failure to add " + key + " to cache " + this.cacheName + ":" + e.getMessage();
163          this.logger.warning(msg);
164        }
165      }
166      
167      /**
168       * Adds the key-value pair to this cache with an explicit expiration time.  
169       * 
170       * @param key The key, typically a UriString.
171       * @param value The value, typically the object returned from the Hackystat service.
172       * @param maxLifeHours The number of hours before this item will expire from cache.
173       */
174      public void put(Serializable key, Serializable value, double maxLifeHours) {
175        try {
176          ElementAttributes attributes = new ElementAttributes();
177          long maxLifeSeconds = (long)(maxLifeHours * 3600D);
178          attributes.setMaxLifeSeconds(maxLifeSeconds);
179          attributes.setIsEternal(false);
180          JCS.getInstance(this.cacheName).putInGroup(key, DEFAULT_GROUP, value, attributes);
181        }
182        catch (CacheException e) {
183          String msg = "Failure to add " + key + " to cache " + this.cacheName + ":" + e.getMessage();
184          this.logger.warning(msg);
185        }
186      }
187      
188      /**
189       * Returns the object associated with key from the cache, or null if not found. 
190       * 
191       * @param key The key whose associated value is to be retrieved.
192       * @return The value, or null if not found.
193       */
194      public Object get(Serializable key) {
195        try {
196          return JCS.getInstance(this.cacheName).getFromGroup(key, DEFAULT_GROUP);
197        }
198        catch (CacheException e) {
199          String msg = "Failure of get: " + key + " in cache " + this.cacheName + ":" + e.getMessage();
200          this.logger.warning(msg);
201          return null;
202        }
203      }
204    
205      /**
206       * Ensures that the key-value pair associated with key is no longer in this cache. 
207       * Logs a message if the cache throws an exception.
208       * 
209       * @param key The key to be removed.
210       */
211      public void remove(Serializable key) {
212        try {
213          JCS.getInstance(this.cacheName).remove(key, DEFAULT_GROUP);
214        }
215        catch (CacheException e) {
216          String msg = "Failure to remove: " + key + " cache " + this.cacheName + ":" + e.getMessage();
217          this.logger.warning(msg);
218        }
219      }
220      
221      /**
222       * Removes everything in the default cache, but not any of the group caches. 
223       */
224      public void clear() {
225        clearGroup(DEFAULT_GROUP);
226      }
227    
228      /**
229       * Clears the default as well as all group caches. 
230       */
231      public void clearAll() {
232        try {
233          JCS.getInstance(this.cacheName).clear();
234        }
235        catch (CacheException e) {
236          String msg = failureMsg + this.cacheName + ":" + e.getMessage();
237          this.logger.warning(msg);
238        }
239      }
240    
241      /**
242       * Returns the set of keys associated with this cache. 
243       * @return The set containing the keys for this cache. 
244       */
245      public Set<Serializable> getKeys() {
246        return getGroupKeys(DEFAULT_GROUP);
247      }
248      
249      /**
250       * Returns the current number of elements in this cache. 
251       * @return The current size of this cache. 
252       */
253      public int size() {
254        return getGroupSize(DEFAULT_GROUP);
255      }
256      
257      
258      /**
259       * Shuts down the specified cache, and removes it from the list of active caches so it can be
260       * created again.
261       * 
262       * @param cacheName The name of the cache to dispose of.
263       */
264      public static void dispose(String cacheName) {
265        try {
266          cacheNames.remove(cacheName);
267          JCS.getInstance(cacheName).dispose();
268        }
269        catch (CacheException e) {
270          String msg = failureMsg + cacheName + ":" + e.getMessage();
271          System.out.println(msg);
272        }
273      }
274      
275      
276      /**
277       * Implements group-based addition of cache elements.
278       * @param key The key.
279       * @param group The group.
280       * @param value The value.
281       */
282      public void putInGroup(Serializable key, String group, Serializable value) {
283        try {
284          JCS.getInstance(this.cacheName).putInGroup(key, group, value);
285        }
286        catch (CacheException e) {
287          String msg = "Failure to add " + key + " to cache " + this.cacheName + ":" + e.getMessage();
288          this.logger.warning(msg);
289        }
290      }
291    
292      /**
293       * Implements group-based retrieval of cache elements. 
294       * @param key The key.
295       * @param group The group.
296       * @return The element associated with key in the group, or null.
297       */
298      public Object getFromGroup(Serializable key, String group) {
299        try {
300          return JCS.getInstance(this.cacheName).getFromGroup(key, group);
301        }
302        catch (CacheException e) {
303          String msg = "Failure of get: " + key + " in cache " + this.cacheName + ":" + e.getMessage();
304          this.logger.warning(msg);
305          return null;
306        }
307      }
308      
309      /**
310       * Implements group-based removal of cache elements. 
311       * @param key The key whose value is to be removed. 
312       * @param group The group.
313       */
314      public void removeFromGroup(Serializable key, String group) {
315        try {
316          JCS.getInstance(this.cacheName).remove(key, group);
317        }
318        catch (CacheException e) {
319          String msg = "Failure to remove: " + key + " cache " + this.cacheName + ":" + e.getMessage();
320          this.logger.warning(msg);
321        }
322      }
323      
324      /**
325       * Returns the set of cache keys associated with this group.
326       * @param group The group.
327       * @return The set of cache keys for this group.
328       */
329      @SuppressWarnings("unchecked")
330      public Set<Serializable> getGroupKeys(String group) {
331        Set<Serializable> keySet;
332        try {
333          keySet = JCS.getInstance(this.cacheName).getGroupKeys(group);
334        }
335        catch (CacheException e) {
336          String msg = "Failure to obtain keyset for cache: " + this.cacheName;
337          this.logger.warning(msg);
338          keySet = new HashSet<Serializable>();
339        }
340        return keySet;
341      }
342      
343      /**
344       * Returns the current number of elements in this cache group.
345       * @param group The name of the group.  
346       * @return The current size of this cache. 
347       */
348      public int getGroupSize(String group) {
349        return getGroupKeys(group).size();
350      }
351     
352      /**
353       * Removes everything in the specified group.
354       * @param group The group name.  
355       */
356      public void clearGroup(String group) {
357        try {
358          JCS cache = JCS.getInstance(this.cacheName);
359          for (Object key : cache.getGroupKeys(group)) {
360            cache.remove(key, group);
361          }
362        }
363        catch (CacheException e) {
364          String msg = failureMsg + this.cacheName + ":" + e.getMessage();
365          this.logger.warning(msg);
366        }
367      }
368    
369      /**
370       * Sets up the Properties instance for configuring this JCS cache instance. Each UriCache is
371       * defined as a JCS "region". Given a UriCache named "PJ", we create a properties instance whose
372       * contents are similar to the following:
373       * 
374       * <pre>
375       * jcs.region.PJ=DC-PJ
376       * jcs.region.PJ.cacheattributes=org.apache.jcs.engine.CompositeCacheAttributes
377       * jcs.region.PJ.cacheattributes.MaxObjects=[maxCacheCapacity]
378       * jcs.region.PJ.cacheattributes.MemoryCacheName=org.apache.jcs.engine.memory.lru.LRUMemoryCache
379       * jcs.region.PJ.cacheattributes.UseMemoryShrinker=true
380       * jcs.region.PJ.cacheattributes.MaxMemoryIdleTimeSeconds=3600
381       * jcs.region.PJ.cacheattributes.ShrinkerIntervalSeconds=3600
382       * jcs.region.PJ.cacheattributes.MaxSpoolPerRun=500
383       * jcs.region.PJ.elementattributes=org.apache.jcs.engine.ElementAttributes
384       * jcs.region.PJ.elementattributes.IsEternal=false
385       * jcs.region.PJ.elementattributes.MaxLifeSeconds=[maxIdleTime]
386       * jcs.auxiliary.DC-PJ=org.apache.jcs.auxiliary.disk.indexed.IndexedDiskCacheFactory
387       * jcs.auxiliary.DC-PJ.attributes=org.apache.jcs.auxiliary.disk.indexed.IndexedDiskCacheAttributes
388       * jcs.auxiliary.DC-PJ.attributes.DiskPath=[cachePath]
389       * jcs.auxiliary.DC-PJ.attributes.maxKeySize=10000000
390       * </pre>
391       * 
392       * We define cachePath as HackystatHome.getHome()/.hackystat/[cacheSubDir]/cache. This enables a
393       * service a cache name of "dailyprojectdata" and have the cache data put inside its internal
394       * subdirectory.
395       * 
396       * See bottom of: http://jakarta.apache.org/jcs/BasicJCSConfiguration.html for more details.
397       * 
398       * @param cacheName The name of this cache, used to define the region properties.
399       * @param subDir The subdirectory name, used to generate the disk storage directory.
400       * @param maxLifeSeconds The maximum life of instances in the cache in seconds before they expire.
401       * @param maxCapacity The maximum size of this cache.
402       * @return The properties file.
403       */
404      private Properties initJcsProps(String cacheName, String subDir, Long maxLifeSeconds, 
405          Long maxCapacity) {
406        String reg = "jcs.region." + cacheName;
407        String regCacheAtt = reg + ".cacheattributes";
408        String regEleAtt = reg + ".elementattributes";
409        String aux = "jcs.auxiliary.DC-" + cacheName;
410        String auxAtt = aux + ".attributes";
411        String memName = "org.apache.jcs.engine.memory.lru.LRUMemoryCache";
412        String diskAttName = "org.apache.jcs.auxiliary.disk.indexed.IndexedDiskCacheAttributes";
413        Properties props = new Properties();
414        props.setProperty(reg, "DC-" + cacheName);
415        props.setProperty(regCacheAtt, "org.apache.jcs.engine.CompositeCacheAttributes");
416        props.setProperty(regCacheAtt + ".MaxObjects", maxCapacity.toString());
417        props.setProperty(regCacheAtt + ".MemoryCacheName", memName);
418        props.setProperty(regCacheAtt + ".UseMemoryShrinker", "true");
419        props.setProperty(regCacheAtt + ".MaxMemoryIdleTimeSeconds", "3600");
420        props.setProperty(regCacheAtt + ".ShrinkerIntervalSeconds", "3600");
421        props.setProperty(regCacheAtt + ".DiskUsagePatternName", "UPDATE");
422        props.setProperty(regCacheAtt + ".MaxSpoolPerRun", "500");
423        props.setProperty(regEleAtt, "org.apache.jcs.engine.ElementAttributes");
424        props.setProperty(regEleAtt + ".IsEternal", "false");
425        props.setProperty(regEleAtt + ".MaxLifeSeconds", maxLifeSeconds.toString());
426        props.setProperty(aux, "org.apache.jcs.auxiliary.disk.indexed.IndexedDiskCacheFactory");
427        props.setProperty(auxAtt, diskAttName);
428        props.setProperty(auxAtt + ".DiskPath", getCachePath(subDir));
429        props.setProperty(auxAtt + ".maxKeySize", "1000000");
430        return props;
431      }
432      
433      /**
434       * Returns the fully qualified file path to the directory in which the backing store files for
435       * this cache will be placed. Creates the path if it does not already exist.
436       * 
437       * @param cacheSubDir The subdirectory where we want to locate the cache files.
438       * @return The fully qualified file path to the location where we should put the index files.
439       */
440      private String getCachePath(String cacheSubDir) {
441        File path = new File(HackystatUserHome.getHome(), ".hackystat/" + cacheSubDir + "/uricache");
442        boolean dirsOk = path.mkdirs();
443        if (!dirsOk && !path.exists()) {
444          throw new RuntimeException("mkdirs() failed");
445        }
446        return path.getAbsolutePath();
447      }
448      
449      /**
450       * Sets the logging level for this logger to level.
451       * @param level A string indicating the level, such as "FINE", "INFO", "ALL", etc.
452       */
453      public void setLoggingLevel(String level) {
454        HackystatLogger.setLoggingLevel(this.logger, level);
455      }
456    }