001 package org.hackystat.tickertape.ticker.data; 002 003 import java.util.Collections; 004 import java.util.HashMap; 005 import java.util.HashSet; 006 import java.util.List; 007 import java.util.ArrayList; 008 import java.util.Map; 009 import java.util.Set; 010 import javax.xml.datatype.XMLGregorianCalendar; 011 012 import java.util.concurrent.ConcurrentHashMap; 013 import java.util.logging.Logger; 014 015 import org.hackystat.sensorbase.client.SensorBaseClient; 016 import org.hackystat.sensorbase.resource.projects.jaxb.Project; 017 import org.hackystat.sensorbase.resource.sensordata.jaxb.Properties; 018 import org.hackystat.sensorbase.resource.sensordata.jaxb.Property; 019 import org.hackystat.sensorbase.resource.sensordata.jaxb.SensorDataRef; 020 import org.hackystat.sensorbase.resource.sensordata.jaxb.SensorData; 021 import org.hackystat.sensorbase.resource.sensordata.jaxb.SensorDataIndex; 022 import org.hackystat.utilities.tstamp.Tstamp; 023 024 /** 025 * Provides a sliding window of recent project data, along with a record of the last time 026 * a tweet was generated from this data. 027 * @author Philip Johnson 028 */ 029 public class ProjectSensorDataLog { 030 031 /** Maps the time when data was retrieved the list of data of interest. */ 032 Map<XMLGregorianCalendar, List<SensorData>> timestamp2SensorDatas = 033 new ConcurrentHashMap<XMLGregorianCalendar, List<SensorData>>(); 034 Map<XMLGregorianCalendar, Project> timestamp2Project = 035 new ConcurrentHashMap<XMLGregorianCalendar, Project>(); 036 037 private SensorBaseClient client = null; 038 private long maxLifeInMillis; 039 private XMLGregorianCalendar lastUpdate = null; 040 private String projectOwner; 041 private String projectName; 042 private Logger logger; 043 044 private List<SensorData> emptyDataList; 045 046 private Map<String, XMLGregorianCalendar> user2lastTweet = 047 new ConcurrentHashMap<String, XMLGregorianCalendar>(); 048 049 /** 050 * Creates a new ProjectSensorDataLog that maintains a sliding window of data. 051 * @param client The SensorBaseClient used to retrieve the data. 052 * @param maxLife The window size, in hours. 053 * @param projectOwner The project owner. 054 * @param projectName The project name. 055 * @param logger The logger to be used if problems occur. 056 */ 057 public ProjectSensorDataLog(SensorBaseClient client, double maxLife, String projectOwner, 058 String projectName, Logger logger) { 059 this.client = client; 060 this.maxLifeInMillis = (long) (60 * 60 * 1000 * maxLife); 061 this.projectOwner = projectOwner; 062 this.projectName = projectName; 063 this.logger = logger; 064 this.emptyDataList = new ArrayList<SensorData>(); 065 } 066 067 /** 068 * Returns the timestamp corresponding to (Current time - maxLife). 069 * @return The maxlife timestamp. 070 */ 071 private XMLGregorianCalendar getMaxLifeTimestamp() { 072 XMLGregorianCalendar currTime = Tstamp.makeTimestamp(); 073 return Tstamp.incrementMilliseconds(currTime, -1 * this.maxLifeInMillis); 074 } 075 076 /** 077 * Retrieves project SensorDataRefs since the last time it was called. 078 * If this is the first invocation, then retrieves data for an interval corresponding 079 * to maxLife. 080 */ 081 public void update() { 082 // If this is the first update call, set lastUpdate to (now - maxlife). 083 if (lastUpdate == null) { 084 this.lastUpdate = getMaxLifeTimestamp(); 085 } 086 // Now retrieve all sensordata for the interval. 087 XMLGregorianCalendar currTime = Tstamp.makeTimestamp(); 088 SensorDataIndex index = null; 089 try { 090 this.logger.fine(String.format("Updating %s/%s from %s to %s", projectName, projectOwner, 091 lastUpdate, currTime)); 092 index = client.getProjectSensorData(projectOwner, projectName, lastUpdate, currTime); 093 } 094 catch (Exception e) { 095 this.logger.warning("Project Sensor Data request failed: " + e.getMessage()); 096 } 097 // Now update lastUpdate. 098 this.lastUpdate = currTime; 099 100 // Update our data structure with the SensorDataRefs, if any. 101 if ((index == null) || (index.getSensorDataRef() == null)) { 102 this.timestamp2SensorDatas.put(currTime, this.emptyDataList); 103 } 104 else { 105 List<SensorData> sensordata = new ArrayList<SensorData>(); 106 for (SensorDataRef ref : index.getSensorDataRef()) { 107 try { 108 SensorData data = this.client.getSensorData(ref); 109 this.logger.fine("Found: " + formatSensorData(data)); 110 sensordata.add(data); 111 } 112 catch (Exception e) { 113 this.logger.warning("Failed to retrieve sensor data: " + e.getMessage()); 114 } 115 } 116 this.timestamp2SensorDatas.put(currTime, sensordata); 117 } 118 119 // Update the project data structure with the current project definition. 120 Project project; 121 try { 122 project = client.getProject(this.projectOwner, this.projectName); 123 this.timestamp2Project.put(this.lastUpdate, project); 124 } 125 catch (Exception e) { 126 this.logger.warning("Project definition request failed: " + e.getMessage()); 127 } 128 129 // Remove any entries that are older than maxLife 130 XMLGregorianCalendar maxLifeTimestamp = this.getMaxLifeTimestamp(); 131 for (XMLGregorianCalendar tstamp : this.timestamp2SensorDatas.keySet()) { 132 if (Tstamp.lessThan(tstamp, maxLifeTimestamp)) { 133 this.logger.fine(String.format("Removing %s because it is less than maxLife (%s)", 134 tstamp, maxLifeTimestamp)); 135 this.timestamp2SensorDatas.remove(tstamp); 136 this.timestamp2Project.remove(tstamp); 137 } 138 } 139 this.logger.fine(this.toString()); 140 } 141 142 /** 143 * Indicate that a tweet was generated based upon the last received sensor data for the 144 * specified user. 145 * @param user The user who had a tweet generated. 146 */ 147 public void setTweet(String user) { 148 this.user2lastTweet.put(user, this.lastUpdate); 149 } 150 151 /** 152 * Returns the list consisting of the project owner and all members from the last update, 153 * or an empty list if problems occurred. 154 * 155 * @return The (possibly empty) list of project members. 156 */ 157 public List<String> getProjectParticipants() { 158 List<String> participants = new ArrayList<String>(); 159 if (this.timestamp2Project.containsKey(this.lastUpdate)) { 160 Project project = this.timestamp2Project.get(this.lastUpdate); 161 participants.add(project.getOwner()); 162 if (!(project.getMembers() == null)) { 163 for (String member : project.getMembers().getMember()) { 164 participants.add(member); 165 } 166 } 167 } 168 return participants; 169 } 170 171 /** 172 * Indicate if a tweet has been generated at least once based upon data received within 173 * the maxLife interval for the specified user. 174 * @param user The user of interest. 175 * @return True if a tweet has been generated recently. 176 */ 177 public boolean hasRecentTweet(String user) { 178 if (this.user2lastTweet.containsKey(user)) { 179 XMLGregorianCalendar maxLife = this.getMaxLifeTimestamp(); 180 XMLGregorianCalendar lastTweet = this.user2lastTweet.get(user); 181 // If the last tweet is older than maxLife, then return true. 182 return Tstamp.lessThan(maxLife, lastTweet); 183 } 184 return false; 185 } 186 187 /** 188 * Returns the (potentially empty) list of sensordatarefs received during the last update 189 * for the specified user. 190 * @param user The user. 191 * @return The list of sensordatarefs, possibly empty. 192 */ 193 public List<SensorData> getRecentSensorData(String user) { 194 if (this.timestamp2SensorDatas.containsKey(this.lastUpdate)) { 195 List<SensorData> datas = new ArrayList<SensorData>(); 196 for (SensorData data : this.timestamp2SensorDatas.get(this.lastUpdate)) { 197 if (data.getOwner().equals(user)) { 198 datas.add(data); 199 } 200 } 201 return datas; 202 } 203 else { 204 // Should never happen, but just in case. 205 this.logger.warning("Could not find data for last update"); 206 return this.emptyDataList; 207 } 208 } 209 210 /** 211 * Returns true if this user has data from the last update. 212 * @param user The user. 213 * @return True if there is sensor data for this user. 214 */ 215 public boolean hasRecentSensorData(String user) { 216 if (this.timestamp2SensorDatas.containsKey(this.lastUpdate)) { 217 for (SensorData data : this.timestamp2SensorDatas.get(this.lastUpdate)) { 218 if (data.getOwner().equals(user)) { 219 return true; 220 } 221 } 222 } 223 return false; 224 } 225 226 /** 227 * Returns a list of all SensorData in our sliding window of data that was generated by the 228 * given user and is of the given SensorDataType. 229 * @param user The user of interest. 230 * @param sdt The sensor data type of interest. 231 * @return A list of matching SensorData. 232 */ 233 public List<SensorData> getSensorData(String user, String sdt) { 234 List<SensorData> datas = new ArrayList<SensorData>(); 235 for (Map.Entry<XMLGregorianCalendar, List<SensorData>> entry : 236 this.timestamp2SensorDatas.entrySet()) { 237 for (SensorData data : entry.getValue()) { 238 if ((user.equals(data.getOwner())) && 239 (sdt.equals(data.getSensorDataType()))) { 240 datas.add(data); 241 } 242 } 243 } 244 return datas; 245 } 246 247 /** 248 * Returns a list of all SensorData for the given timestamp that was generated by the 249 * given user and is of the given SensorDataType. 250 * @param user The user of interest. 251 * @param sdt The sensor data type of interest. 252 * @param timestamp The timestamp of interest. 253 * @return A list of matching SensorData. 254 */ 255 public List<SensorData> getSensorData(String user, String sdt, XMLGregorianCalendar timestamp) { 256 List<SensorData> datas = new ArrayList<SensorData>(); 257 for (SensorData data : this.timestamp2SensorDatas.get(timestamp)) { 258 if ((user.equals(data.getOwner())) && 259 (sdt.equals(data.getSensorDataType()))) { 260 datas.add(data); 261 } 262 } 263 return datas; 264 } 265 266 /** 267 * Returns the set of all SensorDataType names associated with data for this user anywhere in 268 * this log. 269 * @param user The user of interest. 270 * @return A set of strings containing sensor data type names. 271 */ 272 public Set<String> getSensorDataTypes(String user) { 273 Set<String> sdts = new HashSet<String>(); 274 for (Map.Entry<XMLGregorianCalendar, List<SensorData>> entry : 275 this.timestamp2SensorDatas.entrySet()) { 276 for (SensorData data : entry.getValue()) { 277 if (user.equals(data.getOwner())) { 278 sdts.add(data.getSensorDataType()); 279 } 280 } 281 } 282 return sdts; 283 } 284 285 /** 286 * Returns the set of all SensorDataType names associated with data for this user anywhere in 287 * this log. 288 * @param user The user of interest. 289 * @param timestamp The timestamp. 290 * @return A set of strings containing sensor data type names. 291 */ 292 public Set<String> getSensorDataTypes(String user, XMLGregorianCalendar timestamp) { 293 Set<String> sdts = new HashSet<String>(); 294 for (SensorData data : this.timestamp2SensorDatas.get(timestamp)) { 295 if (user.equals(data.getOwner())) { 296 sdts.add(data.getSensorDataType()); 297 } 298 } 299 return sdts; 300 } 301 302 /** 303 * Returns the set of all sensor data owners in this log. 304 * @return A set of strings containing owner emails. 305 */ 306 public Set<String> getOwners() { 307 Set<String> owners = new HashSet<String>(); 308 for (Map.Entry<XMLGregorianCalendar, List<SensorData>> entry : 309 this.timestamp2SensorDatas.entrySet()) { 310 for (SensorData data : entry.getValue()) { 311 owners.add(data.getOwner()); 312 } 313 } 314 return owners; 315 } 316 317 /** 318 * Returns the (possibly empty) set of all sensor data owners in this log for the given time. 319 * Note that no checking is done to see that the timestamp exists in the log. 320 * @param timestamp The timestamp. 321 * @return A set of strings containing owner emails. 322 */ 323 public Set<String> getOwners(XMLGregorianCalendar timestamp) { 324 Set<String> owners = new HashSet<String>(); 325 for (SensorData data : this.timestamp2SensorDatas.get(timestamp)) { 326 owners.add(data.getOwner()); 327 } 328 return owners; 329 } 330 331 332 333 334 /** 335 * Returns true if there is at least one SensorDataRef in our sliding window of data that 336 * was generated by the given user and is of the given SensorDataType. 337 * @param user The user of interest. 338 * @param sdt The sensor data type of interest. 339 * @return True if data of the specified type is present. 340 */ 341 public boolean hasSensorData(String user, String sdt) { 342 for (Map.Entry<XMLGregorianCalendar, List<SensorData>> entry : 343 this.timestamp2SensorDatas.entrySet()) { 344 for (SensorData data : entry.getValue()) { 345 if ((user.equals(data.getOwner())) && 346 (sdt.equals(data.getSensorDataType()))) { 347 return true; 348 } 349 } 350 } 351 return false; 352 } 353 354 /** 355 * Returns true if there is any data in our sliding window. 356 * @return True if data exists. 357 */ 358 public boolean hasSensorData() { 359 for (Map.Entry<XMLGregorianCalendar, List<SensorData>> entry : 360 this.timestamp2SensorDatas.entrySet()) { 361 if (!entry.getValue().isEmpty()) { 362 return true; 363 } 364 } 365 return false; 366 } 367 368 369 /** 370 * Returns a count of the number of distinct files for which DevEvent sensor data 371 * has been generated in the current sliding window of data. 372 * Requires one HTTP call per DevEvent. 373 * @param user The user of interest. 374 * @return The number of files associated with DevEvents during this interval. 375 */ 376 public int getNumFilesWorkedOn(String user) { 377 List<SensorData> datas = getSensorData(user, "DevEvent"); 378 Set<String> files = new HashSet<String>(); 379 if (!datas.isEmpty()) { 380 try { 381 for (SensorData data : datas) { 382 files.add(data.getResource()); 383 } 384 } 385 catch (Exception e) { 386 this.logger.warning("Error occurred retrieving sensor data: " + e.getMessage()); 387 } 388 } 389 return files.size(); 390 } 391 392 /** 393 * Returns a count of the number of sensor data instances of the given type. 394 * @param user The user of interest. 395 * @param sdt The sensor data type. 396 * @return The number of instances of this sensor data type. 397 */ 398 public int getSensorDataCount(String user, String sdt) { 399 return getSensorData(user, sdt).size(); 400 } 401 402 /** 403 * Returns a count of the number of sensor data instances of the given type for the given tstamp. 404 * @param user The user of interest. 405 * @param sdt The sensor data type. 406 * @param timestamp The timestamp. 407 * @return The number of instances of this sensor data type. 408 */ 409 public int getSensorDataCount(String user, String sdt, XMLGregorianCalendar timestamp) { 410 return getSensorData(user, sdt, timestamp).size(); 411 } 412 413 /** 414 * Returns the number of successful builds. 415 * @param user The user. 416 * @return The numnber of builds that were successful. 417 */ 418 public int getBuildSuccessCount(String user) { 419 List<SensorData> datas = getSensorData(user, "Build"); 420 int success = 0; 421 for (SensorData data : datas) { 422 String result = this.getPropertyValue(data, "Result"); 423 if ("Success".equals(result)) { 424 success++; 425 } 426 } 427 return success; 428 } 429 430 /** 431 * Returns the number of passing tests. 432 * @param user The user. 433 * @return The number of test invocations that passed. 434 */ 435 public int getTestPassCount(String user) { 436 List<SensorData> datas = getSensorData(user, "UnitTest"); 437 int success = 0; 438 for (SensorData data : datas) { 439 String result = this.getPropertyValue(data, "Result"); 440 if ("pass".equalsIgnoreCase(result)) { 441 success++; 442 } 443 } 444 return success; 445 } 446 447 /** 448 * Returns a string containing a list of comma separated tool names, or null if no tools. 449 * @param user The user of interest. 450 * @return The tool list, or null. 451 */ 452 public String getToolString(String user) { 453 Set<String> tools = new HashSet<String>(); 454 for (Map.Entry<XMLGregorianCalendar, List<SensorData>> entry : 455 this.timestamp2SensorDatas.entrySet()) { 456 for (SensorData data : entry.getValue()) { 457 if (user.equals(data.getOwner())) { 458 tools.add(data.getTool()); 459 } 460 } 461 } 462 if (tools.size() == 0) { //NOPMD Java 5 compatibility. 463 return null; 464 } 465 StringBuffer buff = new StringBuffer(); 466 for (String tool : tools) { 467 buff.append(tool).append(", "); 468 } 469 // Return the string without the final ',' 470 return buff.toString().substring(0, buff.toString().lastIndexOf(',')); 471 } 472 473 /** 474 * Returns the file that the user worked on the most during the sliding window of data. 475 * Requires one HTTP call per DevEvent. 476 * @param user The user. 477 * @return The file they worked on most, or null if no DevEvent data. 478 */ 479 public String mostWorkedOnFile(String user) { 480 List<SensorData> datas = getSensorData(user, "DevEvent"); 481 Map<String, Integer> file2NumOccurrences = new HashMap<String, Integer>(); 482 if (datas.isEmpty()) { 483 return null; 484 } 485 try { 486 for (SensorData data : datas) { 487 String file = data.getResource(); 488 if (!file2NumOccurrences.containsKey(file)) { 489 file2NumOccurrences.put(file, 0); 490 } 491 int currOccurrences = file2NumOccurrences.get(file); 492 file2NumOccurrences.put(file, currOccurrences + 1); 493 } 494 } 495 catch (Exception e) { 496 this.logger.warning("Error occurred retrieving sensor data: " + e.getMessage()); 497 } 498 // Now find the one that occurred most frequently. 499 int mostOccurred = 0; 500 String mostOccurredFile = "UnknownFile"; 501 for (Map.Entry<String, Integer> entry : file2NumOccurrences.entrySet()) { 502 if (entry.getValue() > mostOccurred) { 503 mostOccurred = entry.getValue(); 504 mostOccurredFile = entry.getKey(); 505 } 506 } 507 // Now return the mostOccurredFile's file name. 508 return getFileName(mostOccurredFile); 509 } 510 511 512 /** 513 * Returns the file name associated with filePath. 514 * @param filePath The file path. 515 * @return The file name. 516 */ 517 private String getFileName(String filePath) { 518 int sepPos = Math.max(filePath.lastIndexOf('/'), filePath.lastIndexOf('\\')); 519 return (sepPos >= 0) ? filePath.substring(sepPos + 1) : filePath; 520 } 521 522 /** 523 * Returns a string with info about the passed SensorData instance. 524 * @param data The sensor data instance. 525 * @return The formatted string. 526 */ 527 private String formatSensorData(SensorData data) { 528 String shortResource = (data.getResource().length() < 20) ? 529 data.getResource() : 530 "..." + data.getResource().substring(data.getResource().length() - 20); 531 String info = String.format("<%s %s %s %s %s>", 532 data.getTimestamp(), data.getOwner(), data.getSensorDataType(), data.getTool(), 533 shortResource); 534 return info; 535 } 536 537 /** 538 * Gets the value for the given property name from the <code>Properties</code> object 539 * contained in the given sensor data instance. 540 * 541 * @param data The sensor data instance to get the property from. 542 * @param propertyName The name of the property to get the value for. 543 * @return Returns the value of the property or null if no matching property was found. 544 */ 545 private String getPropertyValue(SensorData data, String propertyName) { 546 Properties properties = data.getProperties(); 547 if (properties != null) { 548 List<Property> propertyList = properties.getProperty(); 549 for (Property property : propertyList) { 550 if (property.getKey().equals(propertyName)) { 551 return property.getValue(); 552 } 553 } 554 } 555 return null; 556 } 557 558 /** 559 * Provides a formatted string indicating the contents of the this log for debugging purposes. 560 * @return The log as a string. 561 */ 562 @Override 563 public String toString() { 564 StringBuffer buff = new StringBuffer(40); 565 buff.append("\n[ProjectSensorDataLog for: ").append(this.projectName).append('\n'); 566 // Get the timestamps in reverse sorted order. 567 List<XMLGregorianCalendar> sortedTimestamps = Tstamp.sort(this.timestamp2SensorDatas.keySet()); 568 Collections.reverse(sortedTimestamps); 569 for (XMLGregorianCalendar tstamp : sortedTimestamps) { 570 // Print out the timestamp we found in the log. 571 buff.append(tstamp); 572 // Get the owners in this log. 573 Set<String> owners = this.getOwners(tstamp); 574 // For each owner, print out the number of SDTs for this tstamp. 575 for (String owner : owners) { 576 buff.append(" (").append(owner).append(' '); 577 Set<String> sdts = this.getSensorDataTypes(owner, tstamp); 578 for (String sdt : sdts) { 579 buff.append(sdt).append(':').append(this.getSensorDataCount(owner, sdt, tstamp)) 580 .append(' '); 581 } 582 buff.append(')'); 583 } 584 buff.append('\n'); 585 } 586 buff.append(']'); 587 return buff.toString(); 588 } 589 590 /** 591 * Returns the name of the project monitored in this log. 592 * @return This project name. 593 */ 594 public String getProjectName() { 595 return this.projectName; 596 } 597 }