001    package org.hackystat.sensor.ant.issue;
002    
003    import java.io.IOException;
004    import java.io.InputStreamReader;
005    import java.io.Reader;
006    import java.net.MalformedURLException;
007    import java.net.URL;
008    import java.net.URLConnection;
009    import java.text.DateFormat;
010    import java.text.SimpleDateFormat;
011    import java.util.ArrayList;
012    import java.util.HashMap;
013    import java.util.List;
014    import java.util.Locale;
015    import java.util.Map;
016    import javax.xml.datatype.XMLGregorianCalendar;
017    import org.apache.tools.ant.BuildException;
018    import org.apache.tools.ant.Task;
019    import org.hackystat.sensorbase.client.SensorBaseClient;
020    import org.hackystat.sensorbase.client.SensorBaseClientException;
021    import org.hackystat.sensorbase.resource.sensordata.jaxb.SensorData;
022    import org.hackystat.sensorbase.resource.sensordata.jaxb.SensorDataRef;
023    import org.hackystat.sensorshell.SensorShell;
024    import org.hackystat.sensorshell.SensorShellException;
025    import org.hackystat.sensorshell.SensorShellProperties;
026    import org.hackystat.sensorshell.usermap.SensorShellMap;
027    import org.hackystat.sensorshell.usermap.SensorShellMapException;
028    import org.hackystat.utilities.tstamp.Tstamp;
029    import au.com.bytecode.opencsv.CSVReader;
030    
031    /**
032     * Ant task to retieve the issue changes and send those information to Hackystat
033     * server.
034     * 
035     * @author Shaoxuan Zhang
036     *
037     */
038    public class IssueSensor extends Task {
039    
040      /** DateFormat for google csv table. */
041      public static final DateFormat googleDateFormat = 
042        new SimpleDateFormat("MMM dd, yyyy kk:mm:ss", Locale.US);
043      
044      /** SensorDataType of Issue data. */
045      public static final String ISSUE_SENSOR_DATA_TYPE = "Issue";
046      private String tool = "GoogleProjectHosting";
047      private String projectName;
048      private String dataOwnerHackystatAccount = "";
049      private String dataOwnerHackystatPassword = "";
050      private String hackystatSensorbase = "";
051      private boolean isVerbose = false;
052      private XMLGregorianCalendar runTimestamp = Tstamp.makeTimestamp();
053      
054      List<IssueEntry> updatedIssues = new ArrayList<IssueEntry>();
055      Map<String, IssueEntry> issues = new HashMap<String, IssueEntry>();
056      SensorShellMap shellMap;
057      
058      /**
059       * Prepare sensor shell map for this sensor.
060       * @throws SensorShellMapException if error when loading sensor shell map
061       */
062      public IssueSensor() throws SensorShellMapException {
063        //Construct SensorShellMap.
064        shellMap = new SensorShellMap(this.tool);
065        if (this.isVerbose) {
066          System.out.println("Checking for user maps at: " + shellMap.getUserMapFile());
067          System.out.println(this.tool + " accounts found: " + shellMap.getToolAccounts(this.tool));
068        }
069        try {
070          shellMap.validateHackystatInfo(this.tool);
071        }
072        catch (Exception e) {
073          System.out.println("Warning: UserMap validation failed: " + e.getMessage());
074        }
075      }
076      
077      /**
078       * Extracts issue information from feeds, and sends them to the
079       * Hackystat server.
080       * 
081       * @throws BuildException If the task fails.
082       */
083      @Override
084      public void execute() throws BuildException {
085        this.validateProperties(); // sanity check.
086        if (this.isVerbose) {
087          System.out.println("Processing issue updates for project " + this.projectName);
088        }  
089        
090    
091        List<String[]> issueTableContents = new ArrayList<String[]>();
092        //[1] Extract data from Issue CSV table. 
093        if (this.isVerbose) {
094          System.out.println("Retrieving issue csv table from " + this.getAllCsvUrl());
095        }
096        try {
097          URL url = new URL(this.getAllCsvUrl());
098          URLConnection urlConnection = url.openConnection();
099          urlConnection.connect();
100          Reader reader = new InputStreamReader(url.openStream());
101          CSVReader csvReader = new CSVReader(reader);
102          
103          //skip the first line, the header.
104          String[] line = csvReader.readNext();
105          while ((line = csvReader.readNext()) != null && line.length > 1) {
106            issueTableContents.add(line);
107          }
108        }
109        catch (MalformedURLException e) {
110          throw new BuildException("Source URL malformed.", e);
111        }
112        catch (IOException e) {
113          throw new BuildException("Internet connection error.", e);
114        }
115    
116        this.processGoogleIssueCsvData(issueTableContents);
117        
118      }
119      
120      /**
121       * Process the issues information from csv table contents.
122       * @param issueTableContents the parsed table content, in forms of List of String[].
123       */
124      protected void processGoogleIssueCsvData(List<String[]> issueTableContents) {
125        try {
126          runTimestamp = Tstamp.makeTimestamp();
127          
128          updatedIssues.clear();
129    
130          SensorShellProperties props = new SensorShellProperties(this.hackystatSensorbase,
131              this.dataOwnerHackystatAccount, this.dataOwnerHackystatPassword);
132          SensorShell shell = new SensorShell(props, false, this.tool);
133          SensorBaseClient sensorBaseClient = new SensorBaseClient(this.hackystatSensorbase,
134              this.dataOwnerHackystatAccount, this.dataOwnerHackystatPassword);
135          
136          // Retrieve sensor data.
137          if (this.isVerbose) {
138            System.out.println("Retrieve sensordata from " + this.hackystatSensorbase);
139          }
140          for (SensorDataRef ref : sensorBaseClient.getSensorDataIndex(
141              this.dataOwnerHackystatAccount, ISSUE_SENSOR_DATA_TYPE).getSensorDataRef()) {
142            SensorData sensorData = sensorBaseClient.getSensorData(ref);
143            if (tool.equals(sensorData.getTool())) {
144              IssueEntry issue = new IssueEntry(sensorData);
145              issues.put(getResourceFromId(String.valueOf(issue.getIssueId())), issue);
146            }
147          }
148    
149          if (this.isVerbose) {
150            System.out.println(issues.size() + " sensordata found on sensorbase.");
151          }
152          
153          // Put information into sensordata.
154          for (String[] line : issueTableContents) {
155            try {
156              // Retrieve SensorData according to issue id.
157              int id = Integer.valueOf(line[0]);
158              line[5] = mapToHackystatAccount(line[5]);
159              IssueEntry issue = issues.get(getResourceFromId(String.valueOf(id)));
160              if (issue == null) {
161                issue = new IssueEntry(createSensorData(line));
162                if (this.isVerbose) {
163                  System.out.println("New issue #" + issue.getIssueId() + " found. " + 
164                      printStrings(line));
165                }
166                issue.upToDate(line, runTimestamp, false);
167                issues.put(getResourceFromId(String.valueOf(id)), issue);
168                updatedIssues.add(issue);
169              }
170              else {
171                if (issue.upToDate(line, runTimestamp, isVerbose)) {
172                  if (this.isVerbose) {
173                    System.out.println("Issue #" + issue.getIssueId() + " updated. ");
174                  }
175                  updatedIssues.add(issue);
176                }
177              }
178            }
179            catch (Exception e) {
180              e.printStackTrace();
181            }
182          }
183          if (this.isVerbose) {
184            System.out.println("Found " + issues.size() + " issues, " + 
185                updatedIssues.size() + " updated.");
186          }
187          
188          //Put send updated data.
189          for (IssueEntry issueEntry : updatedIssues) {
190            shell.add(issueEntry.getSensorData());
191          }
192          shell.send();
193          shell.quit();
194        }
195        catch (SensorBaseClientException e) {
196          throw new BuildException("SensorBaseClient error.", e);
197        }
198        catch (SensorShellMapException e) {
199          throw new BuildException("SensorShellMap error.", e);
200        }
201        catch (SensorShellException e) {
202          throw new BuildException("SensorShellException error.", e);
203        }
204        catch (Exception e) {
205          throw new BuildException("Error when constructing issue data.", e);
206        }
207      }
208    
209      /**
210       * Print the array of String, separated by comma.
211       * @param line the array of String
212       * @return the string.
213       */
214      private String printStrings(String[] line) {
215        StringBuffer buffer = new StringBuffer();
216        for (String string : line) {
217          buffer.append(string);
218          buffer.append(", ");
219        }
220        return buffer.toString();
221      }
222      
223      /**
224       * add the udpate information to the issue entry's sensordata.
225       * @param issue the issue entry.
226       * @param updates the IssueUpdates.
227       *//*
228      private void addIssueUpdates(IssueEntry issue, List<IssueUpdate> updates) {
229        boolean modified = false;
230        SensorData issueData = issue.getSensorData();
231        for (IssueUpdate update : updates) {
232          String propertyValue = update.getUpdateValueWithTime();
233          if (update.getTimestamp().compare(issueData.getLastMod()) == DatatypeConstants.GREATER) {
234            issueData.addProperty(update.getUpdateKey(), propertyValue);
235            modified = true;
236          }
237          else {
238            boolean foundDuplicate = false;
239            for (Property property : issueData.getProperties().getProperty()) {
240              if (property.getKey().equals(update.getUpdateKey()) && property.getValue()
241              .equals(propertyValue)) {
242                foundDuplicate = true;
243                break;
244              }
245            }
246            if (!foundDuplicate) {
247              issueData.addProperty(update.getUpdateKey(), propertyValue);
248              modified = true;
249            }
250          }
251        }
252        if (modified && runTimestamp.compare(issueData.getLastMod()) == DatatypeConstants.GREATER) {
253          issueData.setLastMod(runTimestamp);
254        }
255      }
256      */
257      /**
258       * Create a SensorData according to the given issue table column.
259       * @param line The given issue table column contents.
260       * @return A SensorData
261       * @throws Exception if error occurs.
262       */
263      private synchronized SensorData createSensorData(String[] line) throws Exception {
264        SensorData sensorData = new SensorData();
265        sensorData.setOwner(dataOwnerHackystatAccount);
266        sensorData.setTool(tool);
267        sensorData.setRuntime(runTimestamp);
268        sensorData.setTimestamp(Tstamp.makeTimestamp(googleDateFormat.parse(line[6]).getTime()));
269        sensorData.setSensorDataType(ISSUE_SENSOR_DATA_TYPE);
270        sensorData.setLastMod(runTimestamp);
271        sensorData.setResource(this.getResourceFromId(line[0]));
272    
273        sensorData.addProperty(IssueEntry.ID_PROPERTY_KEY, line[0]);
274        
275        return sensorData;
276      }
277    
278    
279      /**
280       * Map the given issue account to hackystat account. 
281       * Return the issue account if no mapping found.
282       * @param issueAccount the issue account.
283       * @return the mapped account.
284       * @throws SensorShellMapException if error when mapping.
285       */
286      private String mapToHackystatAccount(String issueAccount) 
287        throws SensorShellMapException {
288    
289        String hackystatAccount;
290        if (shellMap != null && shellMap.hasUserShell(issueAccount)) {
291          hackystatAccount = 
292            shellMap.getUserShell(issueAccount).getProperties().getSensorBaseUser();
293        }
294        else {
295          hackystatAccount = issueAccount;
296        }
297        return hackystatAccount;
298      }
299      
300    
301      /**
302       * Checks and make sure all properties are set up correctly.
303       * 
304       * @throws BuildException If any error is detected in the property setting.
305       */
306      protected void validateProperties() throws BuildException {
307        if (this.projectName == null || this.projectName.length() == 0) {
308          throw new BuildException("Attribute 'projectName' must be set.");
309        }
310        if (this.dataOwnerHackystatAccount == null || this.dataOwnerHackystatAccount.length() == 0) {
311          throw new BuildException("Attribute 'dataOwnerHackystatAccount' must be set.");
312        }
313        if (this.dataOwnerHackystatPassword == null || this.dataOwnerHackystatPassword.length() == 0) {
314          throw new BuildException("Attribute 'dataOwnerHackystatPassword' must be set.");
315        }
316        if (this.hackystatSensorbase == null || this.hackystatSensorbase.length() == 0) {
317          throw new BuildException("Attribute 'hackystatSensorbase' must be set.");
318        }
319      }
320    
321      /**
322       * Returns the shell associated with the specified author, or null if not found. 
323       * The shellCache is
324       * used to store SensorShell instances associated with the specified user. The
325       * SensorShellMap contains the SensorShell instances built from the
326       * UserMap.xml file. This method should be used to retrieve the SensorShell
327       * instances to avoid the unnecessary creation of SensorShell instances when
328       * sending data for each commit entry. Rather than using a brand new
329       * SensorShell instance, this method finds the correct shell in the map,
330       * cache, or creates a brand new shell to use.
331       * @param shellCache the mapping of author to SensorShell.
332       * @param shellMap the mapping of author to SensorShell created by a usermap
333       * entry.
334       * @param author the author used to retrieve the shell instance.
335       * @return the shell instance associated with the author name.
336       * @throws SensorShellMapException thrown if there is a problem retrieving the
337       * shell instance.
338       * @throws SensorShellException thrown if there is a problem retrieving
339       * the Hackystat host from the v8.sensor.properties file.
340       */
341      /*
342      private SensorShell getShell(Map<String, SensorShell> shellCache, SensorShellMap shellMap,
343          String author) throws SensorShellMapException, SensorShellException {
344        if (shellCache.containsKey(author)) {
345          return shellCache.get(author); // Returns a cached shell instance.
346        }
347        else {
348          // If the shell user mapping has a shell, add it to the shell cache.
349          if (shellMap.hasUserShell(author)) {
350            SensorShell shell = shellMap.getUserShell(author);
351            shellCache.put(author, shell);
352            return shell;
353          }
354          else { // Create a new shell and add it to the cache.
355            if ("".equals(this.defaultHackystatAccount)
356                || "".equals(this.defaultHackystatPassword)
357                || "".equals(this.defaultHackystatSensorbase)) {
358              System.out.println("Warning: A user mapping for the user, " + author
359                  + " was not found and no default Hackystat account login, password, "
360                  + "or server was provided. Data ignored.");
361              return null;
362            }
363            SensorShellProperties props = new SensorShellProperties(this.defaultHackystatSensorbase,
364                this.defaultHackystatAccount, this.defaultHackystatPassword);
365    
366            SensorShell shell = new SensorShell(props, false, this.tool);
367            shellCache.put(author, shell);
368            return shell;
369          }
370        }
371      }*/
372    
373      /**
374       * return resource according to the id.
375       * @param id the id.
376       * @return the resource.
377       */
378      private String getResourceFromId(String id) {
379        return "http://code.google.com/p/" + projectName + "/issues/detail?id=" + id;
380      }
381    
382      /**
383       * @return the csvUrl
384       */
385      public String getAllCsvUrl() {
386        return "http://code.google.com/p/" + projectName + "/issues/csv?can=1&q=&" +
387        "colspec=ID%20Type%20Status%20Priority%20Milestone%20Owner%20Opened%20Closed%20Modified";
388      }
389    
390      /**
391       * @return the csvUrl
392       */
393      public String getOpenCsvUrl() {
394        return "http://code.google.com/p/" + projectName + "/issues/csv";
395      }
396      
397      /**
398       * Sets if verbose mode has been enabled.
399       * @param isVerbose true if verbose mode is enabled, false if not.
400       */
401      public void setVerbose(boolean isVerbose) {
402        this.isVerbose = isVerbose;
403      }
404    
405      /**
406       * Sets the default Hackystat sensorbase server.
407       * @param defaultHackystatSensorbase the default sensorbase server.
408       */
409      public void setDefaultHackystatSensorbase(String defaultHackystatSensorbase) {
410        this.hackystatSensorbase = defaultHackystatSensorbase;
411      }
412      
413      /**
414       * @param projectName the projectName to set
415       */
416      public void setProjectName(String projectName) {
417        this.projectName = projectName;
418      }
419    
420      /**
421       * @param dataOwner the dataOwner account to set
422       */
423      public void setDataOwnerHackystatAccount(String dataOwner) {
424        this.dataOwnerHackystatAccount = dataOwner;
425      }
426    
427      /**
428       * @param password the password to set
429       */
430      public void setDataOwnerHackystatPassword(String password) {
431        this.dataOwnerHackystatPassword = password;
432      }
433    
434    }