001    package org.hackystat.tickertape.ticker.data;
002    
003    import java.util.Collections;
004    import java.util.HashMap;
005    import java.util.HashSet;
006    import java.util.List;
007    import java.util.ArrayList;
008    import java.util.Map;
009    import java.util.Set;
010    import javax.xml.datatype.XMLGregorianCalendar;
011    
012    import java.util.concurrent.ConcurrentHashMap;
013    import java.util.logging.Logger;
014    
015    import org.hackystat.sensorbase.client.SensorBaseClient;
016    import org.hackystat.sensorbase.resource.projects.jaxb.Project;
017    import org.hackystat.sensorbase.resource.sensordata.jaxb.Properties;
018    import org.hackystat.sensorbase.resource.sensordata.jaxb.Property;
019    import org.hackystat.sensorbase.resource.sensordata.jaxb.SensorDataRef;
020    import org.hackystat.sensorbase.resource.sensordata.jaxb.SensorData;
021    import org.hackystat.sensorbase.resource.sensordata.jaxb.SensorDataIndex;
022    import org.hackystat.utilities.tstamp.Tstamp;
023    
024    /**
025     * Provides a sliding window of recent project data, along with a record of the last time 
026     * a tweet was generated from this data. 
027     * @author Philip Johnson
028     */
029    public class ProjectSensorDataLog {
030      
031      /** Maps the time when data was retrieved the list of data of interest. */
032      Map<XMLGregorianCalendar, List<SensorData>> timestamp2SensorDatas = 
033        new ConcurrentHashMap<XMLGregorianCalendar, List<SensorData>>();
034      Map<XMLGregorianCalendar, Project> timestamp2Project =
035        new ConcurrentHashMap<XMLGregorianCalendar, Project>();
036      
037      private SensorBaseClient client = null;
038      private long maxLifeInMillis;
039      private XMLGregorianCalendar lastUpdate = null;
040      private String projectOwner;
041      private String projectName;
042      private Logger logger;
043    
044      private List<SensorData> emptyDataList;
045    
046      private Map<String, XMLGregorianCalendar> user2lastTweet =
047        new ConcurrentHashMap<String, XMLGregorianCalendar>();
048    
049      /**
050       * Creates a new ProjectSensorDataLog that maintains a sliding window of data.
051       * @param client The SensorBaseClient used to retrieve the data. 
052       * @param maxLife The window size, in hours. 
053       * @param projectOwner The project owner. 
054       * @param projectName The project name.
055       * @param logger The logger to be used if problems occur.
056       */
057      public ProjectSensorDataLog(SensorBaseClient client, double maxLife, String projectOwner,
058          String projectName, Logger logger) {
059        this.client = client;
060        this.maxLifeInMillis = (long) (60 * 60 * 1000 * maxLife);
061        this.projectOwner = projectOwner;
062        this.projectName = projectName;
063        this.logger = logger;
064        this.emptyDataList = new ArrayList<SensorData>();
065      }
066      
067      /**
068       * Returns the timestamp corresponding to (Current time - maxLife).
069       * @return The maxlife timestamp.
070       */
071      private XMLGregorianCalendar getMaxLifeTimestamp() {
072        XMLGregorianCalendar currTime = Tstamp.makeTimestamp();
073        return Tstamp.incrementMilliseconds(currTime, -1 * this.maxLifeInMillis);
074      }
075      
076      /**
077       * Retrieves project SensorDataRefs since the last time it was called. 
078       * If this is the first invocation, then retrieves data for an interval corresponding
079       * to maxLife. 
080       */
081      public void update() {
082        // If this is the first update call, set lastUpdate to (now - maxlife).
083        if (lastUpdate == null) {
084          this.lastUpdate = getMaxLifeTimestamp(); 
085        }
086        // Now retrieve all sensordata for the interval.
087        XMLGregorianCalendar currTime = Tstamp.makeTimestamp();
088        SensorDataIndex index = null;
089        try {
090          this.logger.fine(String.format("Updating %s/%s from %s to %s", projectName, projectOwner,
091              lastUpdate, currTime));
092          index = client.getProjectSensorData(projectOwner, projectName, lastUpdate, currTime);
093        }
094        catch (Exception e) {
095          this.logger.warning("Project Sensor Data request failed: " + e.getMessage());
096        }
097        // Now update lastUpdate.
098        this.lastUpdate = currTime;
099        
100        // Update our data structure with the SensorDataRefs, if any.
101        if ((index == null) || (index.getSensorDataRef() == null)) {
102          this.timestamp2SensorDatas.put(currTime, this.emptyDataList);
103        }
104        else {
105          List<SensorData> sensordata = new ArrayList<SensorData>();
106          for (SensorDataRef ref : index.getSensorDataRef()) {
107            try {
108              SensorData data = this.client.getSensorData(ref);
109              this.logger.fine("Found: " + formatSensorData(data));
110              sensordata.add(data);
111            }
112            catch (Exception e) {
113              this.logger.warning("Failed to retrieve sensor data: " + e.getMessage());
114            }
115          }
116          this.timestamp2SensorDatas.put(currTime, sensordata);
117        }
118        
119        // Update the project data structure with the current project definition.
120        Project project;
121        try {
122          project = client.getProject(this.projectOwner, this.projectName);
123          this.timestamp2Project.put(this.lastUpdate, project);
124        }
125        catch (Exception e) {
126          this.logger.warning("Project definition request failed: " + e.getMessage()); 
127        }
128        
129        // Remove any entries that are older than maxLife
130        XMLGregorianCalendar maxLifeTimestamp = this.getMaxLifeTimestamp();
131        for (XMLGregorianCalendar tstamp : this.timestamp2SensorDatas.keySet()) {
132          if (Tstamp.lessThan(tstamp, maxLifeTimestamp)) {
133            this.logger.fine(String.format("Removing %s because it is less than maxLife (%s)", 
134                tstamp, maxLifeTimestamp));
135            this.timestamp2SensorDatas.remove(tstamp);
136            this.timestamp2Project.remove(tstamp);
137          }
138        }
139        this.logger.fine(this.toString());
140      }
141      
142      /**
143       * Indicate that a tweet was generated based upon the last received sensor data for the 
144       * specified user.
145       * @param user The user who had a tweet generated. 
146       */
147      public void setTweet(String user) {
148        this.user2lastTweet.put(user, this.lastUpdate);
149      }
150      
151      /**
152       * Returns the list consisting of the project owner and all members from the last update, 
153       * or an empty list if problems occurred.
154       * 
155       * @return The (possibly empty) list of project members.
156       */
157      public List<String> getProjectParticipants() {
158        List<String> participants = new ArrayList<String>();
159        if (this.timestamp2Project.containsKey(this.lastUpdate)) {
160          Project project = this.timestamp2Project.get(this.lastUpdate);
161          participants.add(project.getOwner());
162          if (!(project.getMembers() == null)) {
163            for (String member : project.getMembers().getMember()) {
164              participants.add(member);
165            }
166          }
167        }
168        return participants;
169      }
170      
171      /**
172       * Indicate if a tweet has been generated at least once based upon data received within 
173       * the maxLife interval for the specified user. 
174       * @param user The user of interest.
175       * @return True if a tweet has been generated recently.
176       */
177      public boolean hasRecentTweet(String user) {
178        if (this.user2lastTweet.containsKey(user)) {
179          XMLGregorianCalendar maxLife = this.getMaxLifeTimestamp();
180          XMLGregorianCalendar lastTweet = this.user2lastTweet.get(user);
181          // If the last tweet is older than maxLife, then return true.
182          return Tstamp.lessThan(maxLife, lastTweet); 
183        }
184        return false;
185      }
186      
187      /**
188       * Returns the (potentially empty) list of sensordatarefs received during the last update 
189       * for the specified user. 
190       * @param user The user.
191       * @return The list of sensordatarefs, possibly empty.
192       */
193      public List<SensorData> getRecentSensorData(String user) {
194        if (this.timestamp2SensorDatas.containsKey(this.lastUpdate)) {
195          List<SensorData> datas = new ArrayList<SensorData>();
196          for (SensorData data : this.timestamp2SensorDatas.get(this.lastUpdate)) {
197            if (data.getOwner().equals(user)) {
198              datas.add(data);
199            }
200          }
201          return datas;
202        }
203        else {
204          // Should never happen, but just in case. 
205          this.logger.warning("Could not find data for last update");
206          return this.emptyDataList;
207        }
208      }
209      
210      /**
211       * Returns true if this user has data from the last update. 
212       * @param user The user.
213       * @return True if there is sensor data for this user.
214       */
215      public boolean hasRecentSensorData(String user) {
216        if (this.timestamp2SensorDatas.containsKey(this.lastUpdate)) {
217          for (SensorData data : this.timestamp2SensorDatas.get(this.lastUpdate)) {
218            if (data.getOwner().equals(user)) {
219              return true;
220            }
221          }
222        }
223        return false;
224      }
225    
226      /**
227       * Returns a list of all SensorData in our sliding window of data that was generated by the 
228       * given user and is of the given SensorDataType. 
229       * @param user The user of interest. 
230       * @param sdt The sensor data type of interest.
231       * @return A list of matching SensorData.
232       */
233      public List<SensorData> getSensorData(String user, String sdt) {
234        List<SensorData> datas = new ArrayList<SensorData>();
235        for (Map.Entry<XMLGregorianCalendar, List<SensorData>> entry : 
236          this.timestamp2SensorDatas.entrySet()) {
237          for (SensorData data : entry.getValue()) {
238            if ((user.equals(data.getOwner())) &&
239                (sdt.equals(data.getSensorDataType()))) {
240              datas.add(data);
241            }
242          }
243        }
244        return datas;
245      }
246      
247      /**
248       * Returns a list of all SensorData for the given timestamp that was generated by the 
249       * given user and is of the given SensorDataType. 
250       * @param user The user of interest. 
251       * @param sdt The sensor data type of interest.
252       * @param timestamp The timestamp of interest.
253       * @return A list of matching SensorData.
254       */
255      public List<SensorData> getSensorData(String user, String sdt, XMLGregorianCalendar timestamp) {
256        List<SensorData> datas = new ArrayList<SensorData>();
257        for (SensorData data : this.timestamp2SensorDatas.get(timestamp)) {
258          if ((user.equals(data.getOwner())) &&
259              (sdt.equals(data.getSensorDataType()))) {
260            datas.add(data);
261          }
262        }
263        return datas;
264      }
265      
266      /**
267       * Returns the set of all SensorDataType names associated with data for this user anywhere in
268       * this log.
269       * @param user The user of interest.
270       * @return A set of strings containing sensor data type names. 
271       */
272      public Set<String> getSensorDataTypes(String user) {
273        Set<String> sdts = new HashSet<String>();
274        for (Map.Entry<XMLGregorianCalendar, List<SensorData>> entry : 
275          this.timestamp2SensorDatas.entrySet()) {
276          for (SensorData data : entry.getValue()) {
277            if (user.equals(data.getOwner())) {
278              sdts.add(data.getSensorDataType());
279            }
280          }
281        }
282        return sdts;
283      }
284      
285      /**
286       * Returns the set of all SensorDataType names associated with data for this user anywhere in
287       * this log.
288       * @param user The user of interest.
289       * @param timestamp The timestamp.
290       * @return A set of strings containing sensor data type names. 
291       */
292      public Set<String> getSensorDataTypes(String user, XMLGregorianCalendar timestamp) {
293        Set<String> sdts = new HashSet<String>();
294        for (SensorData data : this.timestamp2SensorDatas.get(timestamp)) {
295          if (user.equals(data.getOwner())) {
296            sdts.add(data.getSensorDataType());
297          }
298        }
299        return sdts;
300      }
301      
302      /**
303       * Returns the set of all sensor data owners in this log.
304       * @return A set of strings containing owner emails.
305       */
306      public Set<String> getOwners() {
307        Set<String> owners = new HashSet<String>();
308        for (Map.Entry<XMLGregorianCalendar, List<SensorData>> entry : 
309          this.timestamp2SensorDatas.entrySet()) {
310          for (SensorData data : entry.getValue()) {
311            owners.add(data.getOwner());
312          }
313        }
314        return owners;
315      }
316      
317      /**
318       * Returns the (possibly empty) set of all sensor data owners in this log for the given time.
319       * Note that no checking is done to see that the timestamp exists in the log.
320       * @param timestamp The timestamp. 
321       * @return A set of strings containing owner emails.
322       */
323      public Set<String> getOwners(XMLGregorianCalendar timestamp) {
324        Set<String> owners = new HashSet<String>();
325        for (SensorData data : this.timestamp2SensorDatas.get(timestamp)) {
326          owners.add(data.getOwner());
327        }
328        return owners;
329      }
330    
331      
332    
333    
334      /**
335       * Returns true if there is at least one SensorDataRef in our sliding window of data that 
336       * was generated by the given user and is of the given SensorDataType.
337       * @param user The user of interest. 
338       * @param sdt The sensor data type of interest.
339       * @return True if data of the specified type is present. 
340       */
341      public boolean hasSensorData(String user, String sdt) {
342        for (Map.Entry<XMLGregorianCalendar, List<SensorData>> entry : 
343          this.timestamp2SensorDatas.entrySet()) {
344          for (SensorData data : entry.getValue()) {
345            if ((user.equals(data.getOwner())) &&
346                (sdt.equals(data.getSensorDataType()))) {
347              return true;
348            }
349          }
350        }
351        return false; 
352      }
353      
354      /**
355       * Returns true if there is any data in our sliding window.
356       * @return True if data exists.
357       */
358      public boolean hasSensorData() {
359        for (Map.Entry<XMLGregorianCalendar, List<SensorData>> entry : 
360          this.timestamp2SensorDatas.entrySet()) {
361          if (!entry.getValue().isEmpty()) {
362            return true;
363          }
364        }
365        return false; 
366      }
367      
368    
369      /**
370       * Returns a count of the number of distinct files for which DevEvent sensor data 
371       * has been generated in the current sliding window of data.
372       * Requires one HTTP call per DevEvent. 
373       * @param user The user of interest. 
374       * @return The number of files associated with DevEvents during this interval.
375       */
376      public int getNumFilesWorkedOn(String user) {
377        List<SensorData> datas = getSensorData(user, "DevEvent");
378        Set<String> files = new HashSet<String>();
379        if (!datas.isEmpty()) {
380          try {
381            for (SensorData data : datas) {
382              files.add(data.getResource());
383            }
384          }
385          catch (Exception e) {
386            this.logger.warning("Error occurred retrieving sensor data: " + e.getMessage());
387          }
388        }
389        return files.size();
390      }
391      
392      /**
393       * Returns a count of the number of sensor data instances of the given type.
394       * @param user The user of interest.
395       * @param sdt The sensor data type. 
396       * @return The number of instances of this sensor data type.
397       */
398      public int getSensorDataCount(String user, String sdt) {
399        return getSensorData(user, sdt).size();
400      }
401      
402      /**
403       * Returns a count of the number of sensor data instances of the given type for the given tstamp.
404       * @param user The user of interest.
405       * @param sdt The sensor data type. 
406       * @param timestamp The timestamp.
407       * @return The number of instances of this sensor data type.
408       */
409      public int getSensorDataCount(String user, String sdt, XMLGregorianCalendar timestamp) {
410        return getSensorData(user, sdt, timestamp).size();
411      }
412      
413      /**
414       * Returns the number of successful builds.
415       * @param user The user.
416       * @return The numnber of builds that were successful.
417       */
418      public int getBuildSuccessCount(String user) {
419        List<SensorData> datas = getSensorData(user, "Build");
420        int success = 0;
421        for (SensorData data : datas) {
422          String result = this.getPropertyValue(data, "Result");
423          if ("Success".equals(result)) {
424            success++;
425          }
426        }
427        return success;
428      }
429      
430      /**
431       * Returns the number of passing tests. 
432       * @param user The user.
433       * @return The number of test invocations that passed.
434       */
435      public int getTestPassCount(String user) {
436        List<SensorData> datas = getSensorData(user, "UnitTest");
437        int success = 0;
438        for (SensorData data : datas) {
439          String result = this.getPropertyValue(data, "Result");
440          if ("pass".equalsIgnoreCase(result)) {
441            success++;
442          }
443        }
444        return success;
445      }
446      
447      /**
448       * Returns a string containing a list of comma separated tool names, or null if no tools.
449       * @param user The user of interest. 
450       * @return The tool list, or null.
451       */
452      public String getToolString(String user) {
453        Set<String> tools = new HashSet<String>();
454        for (Map.Entry<XMLGregorianCalendar, List<SensorData>> entry : 
455          this.timestamp2SensorDatas.entrySet()) {
456          for (SensorData data : entry.getValue()) {
457            if (user.equals(data.getOwner())) {
458              tools.add(data.getTool());
459            }
460          }
461        }
462        if (tools.size() == 0) { //NOPMD Java 5 compatibility.
463          return null;
464        }
465        StringBuffer buff = new StringBuffer();
466        for (String tool : tools) {
467          buff.append(tool).append(", ");
468        }
469        // Return the string without the final ','
470        return buff.toString().substring(0, buff.toString().lastIndexOf(','));
471      }
472      
473      /**
474       * Returns the file that the user worked on the most during the sliding window of data. 
475       * Requires one HTTP call per DevEvent. 
476       * @param user The user.
477       * @return The file they worked on most, or null if no DevEvent data. 
478       */
479      public String mostWorkedOnFile(String user) {
480        List<SensorData> datas = getSensorData(user, "DevEvent");
481        Map<String, Integer> file2NumOccurrences = new HashMap<String, Integer>();
482        if (datas.isEmpty()) {
483          return null;
484        }
485        try {
486          for (SensorData data : datas) {
487            String file = data.getResource();
488            if (!file2NumOccurrences.containsKey(file)) {
489              file2NumOccurrences.put(file, 0);
490            }
491            int currOccurrences = file2NumOccurrences.get(file);
492            file2NumOccurrences.put(file, currOccurrences + 1);
493          }
494        }
495        catch (Exception e) {
496          this.logger.warning("Error occurred retrieving sensor data: " + e.getMessage());
497        }
498        // Now find the one that occurred most frequently. 
499        int mostOccurred = 0;
500        String mostOccurredFile = "UnknownFile";
501        for (Map.Entry<String, Integer> entry : file2NumOccurrences.entrySet()) {
502          if (entry.getValue() > mostOccurred) {
503            mostOccurred = entry.getValue();
504            mostOccurredFile = entry.getKey();
505          }
506        }
507        // Now return the mostOccurredFile's file name.
508        return getFileName(mostOccurredFile);
509      }
510    
511    
512      /**
513       * Returns the file name associated with filePath.
514       * @param filePath The file path. 
515       * @return The file name. 
516       */
517      private String getFileName(String filePath) {
518        int sepPos = Math.max(filePath.lastIndexOf('/'), filePath.lastIndexOf('\\'));
519        return (sepPos >= 0) ? filePath.substring(sepPos + 1) : filePath;
520      }
521     
522      /**
523       * Returns a string with info about the passed SensorData instance. 
524       * @param data The sensor data instance. 
525       * @return The formatted string. 
526       */
527      private String formatSensorData(SensorData data) {
528        String shortResource = (data.getResource().length() < 20) ?
529          data.getResource() :
530            "..." + data.getResource().substring(data.getResource().length() - 20);
531        String info = String.format("<%s %s %s %s %s>",
532            data.getTimestamp(), data.getOwner(), data.getSensorDataType(), data.getTool(), 
533            shortResource);
534        return info;
535      }
536      
537      /**
538       * Gets the value for the given property name from the <code>Properties</code> object
539       * contained in the given sensor data instance.
540       * 
541       * @param data The sensor data instance to get the property from.
542       * @param propertyName The name of the property to get the value for.
543       * @return Returns the value of the property or null if no matching property was found.
544       */
545      private String getPropertyValue(SensorData data, String propertyName) {
546        Properties properties = data.getProperties();
547        if (properties != null) {
548          List<Property> propertyList = properties.getProperty();
549          for (Property property : propertyList) {
550            if (property.getKey().equals(propertyName)) {
551              return property.getValue();
552            }
553          }
554        }
555        return null;
556      }
557      
558      /**
559       * Provides a formatted string indicating the contents of the this log for debugging purposes.
560       * @return The log as a string.
561       */
562      @Override
563      public String toString() {
564        StringBuffer buff = new StringBuffer(40);
565        buff.append("\n[ProjectSensorDataLog for: ").append(this.projectName).append('\n');
566        // Get the timestamps in reverse sorted order.
567        List<XMLGregorianCalendar> sortedTimestamps = Tstamp.sort(this.timestamp2SensorDatas.keySet());
568        Collections.reverse(sortedTimestamps);
569        for (XMLGregorianCalendar tstamp : sortedTimestamps) {
570          // Print out the timestamp we found in the log.
571          buff.append(tstamp);
572          // Get the owners in this log. 
573          Set<String> owners = this.getOwners(tstamp);
574          // For each owner, print out the number of SDTs for this tstamp.
575          for (String owner : owners) {
576            buff.append(" (").append(owner).append(' ');
577            Set<String> sdts = this.getSensorDataTypes(owner, tstamp);
578            for (String sdt : sdts) {
579              buff.append(sdt).append(':').append(this.getSensorDataCount(owner, sdt, tstamp))
580              .append(' ');
581            }
582            buff.append(')');
583          }
584          buff.append('\n');
585        }
586        buff.append(']');
587        return buff.toString();
588      }
589    
590      /**
591       * Returns the name of the project monitored in this log.
592       * @return This project name.
593       */
594      public String getProjectName() {
595        return this.projectName;
596      }
597    }