001 package org.hackystat.sensor.ant.issue; 002 003 import java.io.IOException; 004 import java.io.InputStreamReader; 005 import java.io.Reader; 006 import java.net.MalformedURLException; 007 import java.net.URL; 008 import java.net.URLConnection; 009 import java.text.DateFormat; 010 import java.text.SimpleDateFormat; 011 import java.util.ArrayList; 012 import java.util.HashMap; 013 import java.util.List; 014 import java.util.Locale; 015 import java.util.Map; 016 import javax.xml.datatype.XMLGregorianCalendar; 017 import org.apache.tools.ant.BuildException; 018 import org.apache.tools.ant.Task; 019 import org.hackystat.sensorbase.client.SensorBaseClient; 020 import org.hackystat.sensorbase.client.SensorBaseClientException; 021 import org.hackystat.sensorbase.resource.sensordata.jaxb.SensorData; 022 import org.hackystat.sensorbase.resource.sensordata.jaxb.SensorDataRef; 023 import org.hackystat.sensorshell.SensorShell; 024 import org.hackystat.sensorshell.SensorShellException; 025 import org.hackystat.sensorshell.SensorShellProperties; 026 import org.hackystat.sensorshell.usermap.SensorShellMap; 027 import org.hackystat.sensorshell.usermap.SensorShellMapException; 028 import org.hackystat.utilities.tstamp.Tstamp; 029 import au.com.bytecode.opencsv.CSVReader; 030 031 /** 032 * Ant task to retieve the issue changes and send those information to Hackystat 033 * server. 034 * 035 * @author Shaoxuan Zhang 036 * 037 */ 038 public class IssueSensor extends Task { 039 040 /** DateFormat for google csv table. */ 041 public static final DateFormat googleDateFormat = 042 new SimpleDateFormat("MMM dd, yyyy kk:mm:ss", Locale.US); 043 044 /** SensorDataType of Issue data. */ 045 public static final String ISSUE_SENSOR_DATA_TYPE = "Issue"; 046 private String tool = "GoogleProjectHosting"; 047 private String projectName; 048 private String dataOwnerHackystatAccount = ""; 049 private String dataOwnerHackystatPassword = ""; 050 private String hackystatSensorbase = ""; 051 private boolean isVerbose = false; 052 private XMLGregorianCalendar runTimestamp = Tstamp.makeTimestamp(); 053 054 List<IssueEntry> updatedIssues = new ArrayList<IssueEntry>(); 055 Map<String, IssueEntry> issues = new HashMap<String, IssueEntry>(); 056 SensorShellMap shellMap; 057 058 /** 059 * Prepare sensor shell map for this sensor. 060 * @throws SensorShellMapException if error when loading sensor shell map 061 */ 062 public IssueSensor() throws SensorShellMapException { 063 //Construct SensorShellMap. 064 shellMap = new SensorShellMap(this.tool); 065 if (this.isVerbose) { 066 System.out.println("Checking for user maps at: " + shellMap.getUserMapFile()); 067 System.out.println(this.tool + " accounts found: " + shellMap.getToolAccounts(this.tool)); 068 } 069 try { 070 shellMap.validateHackystatInfo(this.tool); 071 } 072 catch (Exception e) { 073 System.out.println("Warning: UserMap validation failed: " + e.getMessage()); 074 } 075 } 076 077 /** 078 * Extracts issue information from feeds, and sends them to the 079 * Hackystat server. 080 * 081 * @throws BuildException If the task fails. 082 */ 083 @Override 084 public void execute() throws BuildException { 085 this.validateProperties(); // sanity check. 086 if (this.isVerbose) { 087 System.out.println("Processing issue updates for project " + this.projectName); 088 } 089 090 091 List<String[]> issueTableContents = new ArrayList<String[]>(); 092 //[1] Extract data from Issue CSV table. 093 if (this.isVerbose) { 094 System.out.println("Retrieving issue csv table from " + this.getAllCsvUrl()); 095 } 096 try { 097 URL url = new URL(this.getAllCsvUrl()); 098 URLConnection urlConnection = url.openConnection(); 099 urlConnection.connect(); 100 Reader reader = new InputStreamReader(url.openStream()); 101 CSVReader csvReader = new CSVReader(reader); 102 103 //skip the first line, the header. 104 String[] line = csvReader.readNext(); 105 while ((line = csvReader.readNext()) != null && line.length > 1) { 106 issueTableContents.add(line); 107 } 108 } 109 catch (MalformedURLException e) { 110 throw new BuildException("Source URL malformed.", e); 111 } 112 catch (IOException e) { 113 throw new BuildException("Internet connection error.", e); 114 } 115 116 this.processGoogleIssueCsvData(issueTableContents); 117 118 } 119 120 /** 121 * Process the issues information from csv table contents. 122 * @param issueTableContents the parsed table content, in forms of List of String[]. 123 */ 124 protected void processGoogleIssueCsvData(List<String[]> issueTableContents) { 125 try { 126 runTimestamp = Tstamp.makeTimestamp(); 127 128 updatedIssues.clear(); 129 130 SensorShellProperties props = new SensorShellProperties(this.hackystatSensorbase, 131 this.dataOwnerHackystatAccount, this.dataOwnerHackystatPassword); 132 SensorShell shell = new SensorShell(props, false, this.tool); 133 SensorBaseClient sensorBaseClient = new SensorBaseClient(this.hackystatSensorbase, 134 this.dataOwnerHackystatAccount, this.dataOwnerHackystatPassword); 135 136 // Retrieve sensor data. 137 if (this.isVerbose) { 138 System.out.println("Retrieve sensordata from " + this.hackystatSensorbase); 139 } 140 for (SensorDataRef ref : sensorBaseClient.getSensorDataIndex( 141 this.dataOwnerHackystatAccount, ISSUE_SENSOR_DATA_TYPE).getSensorDataRef()) { 142 SensorData sensorData = sensorBaseClient.getSensorData(ref); 143 if (tool.equals(sensorData.getTool())) { 144 IssueEntry issue = new IssueEntry(sensorData); 145 issues.put(getResourceFromId(String.valueOf(issue.getIssueId())), issue); 146 } 147 } 148 149 if (this.isVerbose) { 150 System.out.println(issues.size() + " sensordata found on sensorbase."); 151 } 152 153 // Put information into sensordata. 154 for (String[] line : issueTableContents) { 155 try { 156 // Retrieve SensorData according to issue id. 157 int id = Integer.valueOf(line[0]); 158 line[5] = mapToHackystatAccount(line[5]); 159 IssueEntry issue = issues.get(getResourceFromId(String.valueOf(id))); 160 if (issue == null) { 161 issue = new IssueEntry(createSensorData(line)); 162 if (this.isVerbose) { 163 System.out.println("New issue #" + issue.getIssueId() + " found. " + 164 printStrings(line)); 165 } 166 issue.upToDate(line, runTimestamp, false); 167 issues.put(getResourceFromId(String.valueOf(id)), issue); 168 updatedIssues.add(issue); 169 } 170 else { 171 if (issue.upToDate(line, runTimestamp, isVerbose)) { 172 if (this.isVerbose) { 173 System.out.println("Issue #" + issue.getIssueId() + " updated. "); 174 } 175 updatedIssues.add(issue); 176 } 177 } 178 } 179 catch (Exception e) { 180 e.printStackTrace(); 181 } 182 } 183 if (this.isVerbose) { 184 System.out.println("Found " + issues.size() + " issues, " + 185 updatedIssues.size() + " updated."); 186 } 187 188 //Put send updated data. 189 for (IssueEntry issueEntry : updatedIssues) { 190 shell.add(issueEntry.getSensorData()); 191 } 192 shell.send(); 193 shell.quit(); 194 } 195 catch (SensorBaseClientException e) { 196 throw new BuildException("SensorBaseClient error.", e); 197 } 198 catch (SensorShellMapException e) { 199 throw new BuildException("SensorShellMap error.", e); 200 } 201 catch (SensorShellException e) { 202 throw new BuildException("SensorShellException error.", e); 203 } 204 catch (Exception e) { 205 throw new BuildException("Error when constructing issue data.", e); 206 } 207 } 208 209 /** 210 * Print the array of String, separated by comma. 211 * @param line the array of String 212 * @return the string. 213 */ 214 private String printStrings(String[] line) { 215 StringBuffer buffer = new StringBuffer(); 216 for (String string : line) { 217 buffer.append(string); 218 buffer.append(", "); 219 } 220 return buffer.toString(); 221 } 222 223 /** 224 * add the udpate information to the issue entry's sensordata. 225 * @param issue the issue entry. 226 * @param updates the IssueUpdates. 227 *//* 228 private void addIssueUpdates(IssueEntry issue, List<IssueUpdate> updates) { 229 boolean modified = false; 230 SensorData issueData = issue.getSensorData(); 231 for (IssueUpdate update : updates) { 232 String propertyValue = update.getUpdateValueWithTime(); 233 if (update.getTimestamp().compare(issueData.getLastMod()) == DatatypeConstants.GREATER) { 234 issueData.addProperty(update.getUpdateKey(), propertyValue); 235 modified = true; 236 } 237 else { 238 boolean foundDuplicate = false; 239 for (Property property : issueData.getProperties().getProperty()) { 240 if (property.getKey().equals(update.getUpdateKey()) && property.getValue() 241 .equals(propertyValue)) { 242 foundDuplicate = true; 243 break; 244 } 245 } 246 if (!foundDuplicate) { 247 issueData.addProperty(update.getUpdateKey(), propertyValue); 248 modified = true; 249 } 250 } 251 } 252 if (modified && runTimestamp.compare(issueData.getLastMod()) == DatatypeConstants.GREATER) { 253 issueData.setLastMod(runTimestamp); 254 } 255 } 256 */ 257 /** 258 * Create a SensorData according to the given issue table column. 259 * @param line The given issue table column contents. 260 * @return A SensorData 261 * @throws Exception if error occurs. 262 */ 263 private synchronized SensorData createSensorData(String[] line) throws Exception { 264 SensorData sensorData = new SensorData(); 265 sensorData.setOwner(dataOwnerHackystatAccount); 266 sensorData.setTool(tool); 267 sensorData.setRuntime(runTimestamp); 268 sensorData.setTimestamp(Tstamp.makeTimestamp(googleDateFormat.parse(line[6]).getTime())); 269 sensorData.setSensorDataType(ISSUE_SENSOR_DATA_TYPE); 270 sensorData.setLastMod(runTimestamp); 271 sensorData.setResource(this.getResourceFromId(line[0])); 272 273 sensorData.addProperty(IssueEntry.ID_PROPERTY_KEY, line[0]); 274 275 return sensorData; 276 } 277 278 279 /** 280 * Map the given issue account to hackystat account. 281 * Return the issue account if no mapping found. 282 * @param issueAccount the issue account. 283 * @return the mapped account. 284 * @throws SensorShellMapException if error when mapping. 285 */ 286 private String mapToHackystatAccount(String issueAccount) 287 throws SensorShellMapException { 288 289 String hackystatAccount; 290 if (shellMap != null && shellMap.hasUserShell(issueAccount)) { 291 hackystatAccount = 292 shellMap.getUserShell(issueAccount).getProperties().getSensorBaseUser(); 293 } 294 else { 295 hackystatAccount = issueAccount; 296 } 297 return hackystatAccount; 298 } 299 300 301 /** 302 * Checks and make sure all properties are set up correctly. 303 * 304 * @throws BuildException If any error is detected in the property setting. 305 */ 306 protected void validateProperties() throws BuildException { 307 if (this.projectName == null || this.projectName.length() == 0) { 308 throw new BuildException("Attribute 'projectName' must be set."); 309 } 310 if (this.dataOwnerHackystatAccount == null || this.dataOwnerHackystatAccount.length() == 0) { 311 throw new BuildException("Attribute 'dataOwnerHackystatAccount' must be set."); 312 } 313 if (this.dataOwnerHackystatPassword == null || this.dataOwnerHackystatPassword.length() == 0) { 314 throw new BuildException("Attribute 'dataOwnerHackystatPassword' must be set."); 315 } 316 if (this.hackystatSensorbase == null || this.hackystatSensorbase.length() == 0) { 317 throw new BuildException("Attribute 'hackystatSensorbase' must be set."); 318 } 319 } 320 321 /** 322 * Returns the shell associated with the specified author, or null if not found. 323 * The shellCache is 324 * used to store SensorShell instances associated with the specified user. The 325 * SensorShellMap contains the SensorShell instances built from the 326 * UserMap.xml file. This method should be used to retrieve the SensorShell 327 * instances to avoid the unnecessary creation of SensorShell instances when 328 * sending data for each commit entry. Rather than using a brand new 329 * SensorShell instance, this method finds the correct shell in the map, 330 * cache, or creates a brand new shell to use. 331 * @param shellCache the mapping of author to SensorShell. 332 * @param shellMap the mapping of author to SensorShell created by a usermap 333 * entry. 334 * @param author the author used to retrieve the shell instance. 335 * @return the shell instance associated with the author name. 336 * @throws SensorShellMapException thrown if there is a problem retrieving the 337 * shell instance. 338 * @throws SensorShellException thrown if there is a problem retrieving 339 * the Hackystat host from the v8.sensor.properties file. 340 */ 341 /* 342 private SensorShell getShell(Map<String, SensorShell> shellCache, SensorShellMap shellMap, 343 String author) throws SensorShellMapException, SensorShellException { 344 if (shellCache.containsKey(author)) { 345 return shellCache.get(author); // Returns a cached shell instance. 346 } 347 else { 348 // If the shell user mapping has a shell, add it to the shell cache. 349 if (shellMap.hasUserShell(author)) { 350 SensorShell shell = shellMap.getUserShell(author); 351 shellCache.put(author, shell); 352 return shell; 353 } 354 else { // Create a new shell and add it to the cache. 355 if ("".equals(this.defaultHackystatAccount) 356 || "".equals(this.defaultHackystatPassword) 357 || "".equals(this.defaultHackystatSensorbase)) { 358 System.out.println("Warning: A user mapping for the user, " + author 359 + " was not found and no default Hackystat account login, password, " 360 + "or server was provided. Data ignored."); 361 return null; 362 } 363 SensorShellProperties props = new SensorShellProperties(this.defaultHackystatSensorbase, 364 this.defaultHackystatAccount, this.defaultHackystatPassword); 365 366 SensorShell shell = new SensorShell(props, false, this.tool); 367 shellCache.put(author, shell); 368 return shell; 369 } 370 } 371 }*/ 372 373 /** 374 * return resource according to the id. 375 * @param id the id. 376 * @return the resource. 377 */ 378 private String getResourceFromId(String id) { 379 return "http://code.google.com/p/" + projectName + "/issues/detail?id=" + id; 380 } 381 382 /** 383 * @return the csvUrl 384 */ 385 public String getAllCsvUrl() { 386 return "http://code.google.com/p/" + projectName + "/issues/csv?can=1&q=&" + 387 "colspec=ID%20Type%20Status%20Priority%20Milestone%20Owner%20Opened%20Closed%20Modified"; 388 } 389 390 /** 391 * @return the csvUrl 392 */ 393 public String getOpenCsvUrl() { 394 return "http://code.google.com/p/" + projectName + "/issues/csv"; 395 } 396 397 /** 398 * Sets if verbose mode has been enabled. 399 * @param isVerbose true if verbose mode is enabled, false if not. 400 */ 401 public void setVerbose(boolean isVerbose) { 402 this.isVerbose = isVerbose; 403 } 404 405 /** 406 * Sets the default Hackystat sensorbase server. 407 * @param defaultHackystatSensorbase the default sensorbase server. 408 */ 409 public void setDefaultHackystatSensorbase(String defaultHackystatSensorbase) { 410 this.hackystatSensorbase = defaultHackystatSensorbase; 411 } 412 413 /** 414 * @param projectName the projectName to set 415 */ 416 public void setProjectName(String projectName) { 417 this.projectName = projectName; 418 } 419 420 /** 421 * @param dataOwner the dataOwner account to set 422 */ 423 public void setDataOwnerHackystatAccount(String dataOwner) { 424 this.dataOwnerHackystatAccount = dataOwner; 425 } 426 427 /** 428 * @param password the password to set 429 */ 430 public void setDataOwnerHackystatPassword(String password) { 431 this.dataOwnerHackystatPassword = password; 432 } 433 434 }