001    package org.hackystat.sensor.ant.perforce;
002    
003    import java.io.File;
004    import java.text.ParseException;
005    import java.text.SimpleDateFormat;
006    import java.util.Date;
007    import java.util.HashMap;
008    import java.util.Locale;
009    import java.util.Map;
010    
011    import org.apache.tools.ant.BuildException;
012    import org.apache.tools.ant.Task;
013    import org.hackystat.sensorshell.SensorShell;
014    import org.hackystat.sensorshell.SensorShellException;
015    import org.hackystat.sensorshell.SensorShellProperties;
016    import org.hackystat.sensorshell.usermap.SensorShellMap;
017    import org.hackystat.sensorshell.usermap.SensorShellMapException;
018    import org.hackystat.utilities.email.ValidateEmailSyntax;
019    import org.hackystat.utilities.time.period.Day;
020    import org.hackystat.utilities.tstamp.Tstamp;
021    import org.hackystat.utilities.tstamp.TstampSet;
022    
023    /**
024     * A sensor for collecting Commit information from the Perforce CM system. 
025     * @author Philip Johnson
026     */
027    public class PerforceSensor extends Task {
028      private String depotPath;
029      private String port;
030      private String userName;
031      private String password;
032      private String fileNamePrefix;
033      private String p4ExecutablePath;
034      private String defaultHackystatAccount = "";
035      private String defaultHackystatPassword = "";
036      private String defaultHackystatSensorbase = "";
037      private String p4SysRoot = "C:\\WINDOWS";
038      private String p4SysDrive = "C:";
039      private SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd", Locale.US);
040      private String fromDateString, toDateString;
041      private Date fromDate, toDate;
042      private boolean isVerbose = false;
043      private boolean ignoreWhitespace = false;
044      private String tool = "perforce";
045      
046      /** Initialize a new instance of a PerforceSensor. */
047      public PerforceSensor() {
048        //nothing yet.
049      }
050      
051      /**
052       * Sets the user name to access the Perforce repository. 
053       * @param userName The user name.
054       */
055      public void setUserName(String userName) {
056        this.userName = userName;
057      }
058    
059      /**
060       * Sets if verbose mode has been enabled.
061       * @param isVerbose true if verbose mode is enabled, false if not.
062       */
063      public void setVerbose(boolean isVerbose) {
064        this.isVerbose = isVerbose;
065      }
066    
067      /**
068       * Sets the password for the user name.
069       * 
070       * @param password The password.
071       */
072      public void setPassword(String password) {
073        this.password = password;
074      }
075      
076      /**
077       * True if whitespace changes should be ignored by the underlying Perforce diff2 program
078       * when calculating lines added, changed, and deleted. 
079       * @param ignoreWhitespace True if whitespace changes should be ignored. 
080       */
081      public void setIgnoreWhitespace(boolean ignoreWhitespace) {
082        this.ignoreWhitespace = ignoreWhitespace;
083      }
084    
085      /**
086       * Sets a string to be prepended to the file path in commit metric. Recall
087       * that Perforce sensor gets the depotPath to the file, 
088       * however, most hackystat analysis requires fully qualified file path. This
089       * prefix can be used to turn relative file path into some pseudo fully
090       * qualified file path.
091       * 
092       * @param fileNamePrefix The string to be prepended to the file path in commit
093       * metric.
094       */
095      public void setFileNamePrefix(String fileNamePrefix) {
096        this.fileNamePrefix = fileNamePrefix;
097      }
098    
099      /**
100       * Sets a default Hackystat account to which to send commit data when there is
101       * no Perforce user to Hackystat account mapping.
102       * 
103       * @param defaultHackystatAccount The default Hackystat account.
104       */
105      public void setDefaultHackystatAccount(String defaultHackystatAccount) {
106        this.defaultHackystatAccount = defaultHackystatAccount;
107      }
108    
109      /**
110       * Sets the default Hackystat account password.
111       * @param defaultHackystatPassword the default account password.
112       */
113      public void setDefaultHackystatPassword(String defaultHackystatPassword) {
114        this.defaultHackystatPassword = defaultHackystatPassword;
115      }
116    
117      /**
118       * Sets the default Hackystat sensorbase server.
119       * @param defaultHackystatSensorbase the default sensorbase server.
120       */
121      public void setDefaultHackystatSensorbase(String defaultHackystatSensorbase) {
122        this.defaultHackystatSensorbase = defaultHackystatSensorbase;
123      }
124    
125      /**
126       * Sets the optional fromDate. If fromDate is set, toDate must be set. This
127       * field must be conform to yyyy/MM/dd format.
128       * 
129       * @param fromDateString The first date from which we send commit information
130       * to Hackystat server.
131       */
132      public void setFromDate(String fromDateString) {
133        this.fromDateString = fromDateString;
134      }
135    
136      /**
137       * Sets the optional toDate. If toDate is set, fromDate must be set. This
138       * field must be conform to yyyy/MM/dd format.
139       * 
140       * @param toDateString The last date to which we send commit information to
141       * Hackystat server.
142       */
143      public void setToDate(String toDateString) {
144        this.toDateString = toDateString;
145      }
146      
147      /**
148       * Sets the port to the Perforce server.  Example: "public.perforce.com:1666".
149       * @param port The port. 
150       */
151      public void setPort(String port) {
152        this.port = port;
153      }
154      
155      /**
156       * Sets the path to the p4 executable. Example: "C:\\Program Files\\Perforce\\P4.EXE".
157       * @param path The path.
158       */
159      public void setP4ExecutablePath(String path) {
160        this.p4ExecutablePath = path;
161      }
162      
163      /**
164       * Sets the path to the Window system root dir. Default: "C:\\WINDOWS".
165       * This is needed by the p4 executable on Windows for reasons only it knows.
166       * @param sysroot The sysroot.
167       */
168      public void setP4SysRoot(String sysroot) {
169        this.p4SysRoot = sysroot;
170      }
171      
172      /**
173       * Sets the path to the Window system drive. Default: "C:".
174       * This is needed by the p4 executable on Windows for reasons only it knows.
175       * @param sysdrive The sysdrive.
176       */
177      public void setP4SysDrive(String sysdrive) {
178        this.p4SysDrive = sysdrive;
179      }
180      
181      /**
182       * Sets the depot path in the Perforce server. Example: "//guest/philip_johnson/...".
183       * @param depotPath The depot path. 
184       */
185      public void setDepotPath (String depotPath) {
186        this.depotPath = depotPath;
187      }
188      
189      
190      /**
191       * Checks and make sure all properties are set up correctly.
192       * 
193       * @throws BuildException If any error is detected in the property setting.
194       */
195      private void validateProperties() throws BuildException {
196        if (this.port == null || this.port.length() == 0) {
197          throw new BuildException("Attribute 'port' must be set.");
198        }
199        if (this.depotPath == null || this.depotPath.length() == 0) {
200          throw new BuildException("Attribute 'repositoryUrl' must be set.");
201        }
202        if (this.userName == null || this.userName.length() == 0) {
203          throw new BuildException("Attribute 'userName' must be set.");
204        }
205        if (this.password == null || this.password.length() == 0) {
206          throw new BuildException("Attribute 'password' must be set.");
207        }
208        if (this.depotPath == null || this.depotPath.length() == 0) {
209          throw new BuildException("Attribute 'repositoryUrl' must be set.");
210        }
211        if (this.p4ExecutablePath == null || this.p4ExecutablePath.length() == 0) {
212          throw new BuildException("Attribute 'p4ExecutablePath' must be set.");
213        }
214        File p4Executable = new File(this.p4ExecutablePath);
215        if (!p4Executable.exists()) {
216          throw new BuildException("Attribute 'p4ExecutablePath' " + this.p4ExecutablePath +
217              " does not appear to point to an actual file.");
218        }
219        
220        // If default* is specified, then all should be specified. 
221        if (((this.defaultHackystatAccount != null) || 
222             (this.defaultHackystatPassword != null) ||
223             (this.defaultHackystatSensorbase != null)) &&
224            ((this.defaultHackystatAccount == null) || 
225             (this.defaultHackystatPassword == null) ||
226             (this.defaultHackystatSensorbase == null))) {
227          throw new BuildException ("If one of default Hackystat account, password, or sensorbase " +
228              "is specified, then all must be specified.");
229        }
230        
231        // Check to make sure that defaultHackystatAccount looks like a real email address.
232        if (!ValidateEmailSyntax.isValid(this.defaultHackystatAccount)) {
233          throw new BuildException("Attribute 'defaultHackystatAccount' " + this.defaultHackystatAccount
234              + " does not appear to be a valid email address.");
235        }
236        
237        // If fromDate and toDate not set, we extract commit information for the previous day.
238        if (this.fromDateString == null && this.toDateString == null) {
239          Day previousDay = Day.getInstance().inc(-1);
240          this.fromDate = new Date(previousDay.getFirstTickOfTheDay() - 1);
241          this.toDate = new Date(previousDay.getLastTickOfTheDay());
242        }
243        else {
244          try {
245            if (this.hasSetToAndFromDates()) {
246              this.fromDate = new Date(Day.getInstance(this.dateFormat.parse(this.fromDateString))
247                  .getFirstTickOfTheDay() - 1);
248              this.toDate = new Date(Day.getInstance(this.dateFormat.parse(this.toDateString))
249                  .getLastTickOfTheDay());
250            }
251            else {
252              throw new BuildException(
253                  "Attributes 'fromDate' and 'toDate' must either be both set or both not set.");
254            }
255          }
256          catch (ParseException ex) {
257            throw new BuildException("Unable to parse 'fromDate' or 'toDate'.", ex);
258          }
259    
260          if (this.fromDate.compareTo(this.toDate) > 0) {
261            throw new BuildException("Attribute 'fromDate' must be a date before 'toDate'.");
262          }
263        }
264      }
265    
266      /**
267       * Returns true if both of the to and from date strings have been set by the
268       * client. Both dates must be set or else this sensor will not know which
269       * revisions to grab commit information.
270       * @return true if both the to and from date strings have been set.
271       */
272      private boolean hasSetToAndFromDates() {
273        return (this.fromDateString != null) && (this.toDateString != null);
274      }
275    
276    
277      /**
278       * Extracts commit information from Perforce server, and sends them to the Hackystat server.
279       * 
280       * @throws BuildException If the task fails.
281       */
282      @Override
283      public void execute() throws BuildException {
284        this.validateProperties(); // sanity check.
285        if (this.isVerbose) {
286          System.out.printf("Processing changelists for %s %s between %s and %s. %n",
287              this.port, this.depotPath, this.fromDate, this.toDate);
288        }
289    
290        try {
291          Map<String, SensorShell> shellCache = new HashMap<String, SensorShell>();
292          SensorShellMap shellMap = new SensorShellMap(this.tool);
293          if (this.isVerbose) {
294            System.out.println("Checking for user maps at: " + shellMap.getUserMapFile());
295            System.out.println("Perforce accounts found: " + shellMap.getToolAccounts(this.tool));
296          }
297          try {
298            shellMap.validateHackystatInfo(this.tool);
299          }
300          catch (Exception e) {
301            System.out.println("Warning: UserMap validation failed: " + e.getMessage());
302          }
303          
304          P4Environment p4Env = new P4Environment();
305          p4Env.setP4Port(this.port);
306          p4Env.setP4User(this.userName);
307          p4Env.setP4Password(this.password);
308          p4Env.setP4Executable(this.p4ExecutablePath);
309          // These are given default values above.  User need not set them in the Ant task unless
310          // the defaults are not correct.
311          p4Env.setP4SystemDrive(this.p4SysDrive);
312          p4Env.setP4SystemRoot(this.p4SysRoot);
313          p4Env.setVerbose(false); // could set this to true for lots of p4 debugging output. 
314          PerforceCommitProcessor processor = new PerforceCommitProcessor(p4Env, this.depotPath);
315          processor.setIgnoreWhitespace(this.ignoreWhitespace);
316          processor.processChangeLists(dateFormat.format(this.fromDate), 
317              dateFormat.format(this.toDate));
318          int entriesAdded = 0;
319          TstampSet tstampSet = new TstampSet();
320          for (PerforceChangeListData data : processor.getChangeListDataList()) {
321            if (this.isVerbose) {
322              System.out.printf("Retrieved Perforce changelist: %d%n", data.getId());
323            }
324            String author = data.getOwner();
325            Date commitTime = data.getModTime();
326            for (PerforceChangeListData.PerforceFileData fileData : data.getFileData()) {
327              SensorShell shell = this.getShell(shellCache, shellMap, author);
328              this.processCommitEntry(shell, author, tstampSet
329                  .getUniqueTstamp(commitTime.getTime()), commitTime, data.getId(), fileData);
330              entriesAdded++;
331            }
332          }
333          // Always make sure you call cleanup() at the end. 
334          processor.cleanup();
335          if (this.isVerbose) {
336            System.out.println("Found " + entriesAdded + " commit records.");
337          }
338    
339          // Send the sensor data after all entries have been processed.
340          for (SensorShell shell : shellCache.values()) {
341            if (this.isVerbose) {
342              System.out.println("Sending data to " + shell.getProperties().getSensorBaseUser() + 
343                  " at " + shell.getProperties().getSensorBaseHost());
344            }
345            shell.send();
346            shell.quit();
347          }
348        }
349        catch (Exception ex) {
350          throw new BuildException(ex);
351        }
352      }
353    
354      /**
355       * Returns the shell associated with the specified author. The shellCache is
356       * used to store SensorShell instances associated with the specified user. The
357       * SensorShellMap contains the SensorShell instances built from the
358       * UserMap.xml file. This method should be used to retrieve the SensorShell
359       * instances to avoid the unnecessary creation of SensorShell instances when
360       * sending data for each commit entry. Rather than using a brand new
361       * SensorShell instance, this method finds the correct shell in the map,
362       * cache, or creates a brand new shell to use.
363       * @param shellCache the mapping of author to SensorShell.
364       * @param shellMap the mapping of author to SensorShell created by a usermap
365       * entry.
366       * @param author the author used to retrieve the shell instance.
367       * @return the shell instance associated with the author name.
368       * @throws SensorShellMapException thrown if there is a problem retrieving the
369       * shell instance.
370       * @throws SensorShellException thrown if there is a problem retrieving
371       * the Hackystat host from the v8.sensor.properties file.
372       */
373      private SensorShell getShell(Map<String, SensorShell> shellCache, SensorShellMap shellMap,
374          String author) throws SensorShellMapException, SensorShellException {
375        if (shellCache.containsKey(author)) {
376          return shellCache.get(author); // Returns a cached shell instance.
377        }
378        else {
379          // If the shell user mapping has a shell, add it to the shell cache.
380          if (shellMap.hasUserShell(author)) {
381            SensorShell shell = shellMap.getUserShell(author);
382            shellCache.put(author, shell);
383            return shell;
384          }
385          else { // Create a new shell and add it to the cache.
386            if ("".equals(this.defaultHackystatAccount)
387                || "".equals(this.defaultHackystatPassword)
388                || "".equals(this.defaultHackystatSensorbase)) {
389              throw new BuildException("A user mapping for the user, " + author
390                  + " was not found and no default Hackystat account login, password, "
391                  + "or server was provided.");
392            }
393            SensorShellProperties props = new SensorShellProperties(this.defaultHackystatSensorbase,
394                this.defaultHackystatAccount, this.defaultHackystatPassword);
395    
396            SensorShell shell = new SensorShell(props, false, "svn");
397            shellCache.put(author, shell);
398            return shell;
399          }
400        }
401      }
402    
403      /**
404       * Processes a fileData record entry and extracts relevant metrics.
405       * 
406       * @param shell The shell that the commit record information is added to.
407       * @param author The author of the commit.
408       * @param timestamp the unique timestamp that is associated with the specified entry.
409       * @param commitTime The commit time.
410       * @param revision The changelist ID number.
411       * @param fileData The fileData.
412       * 
413       * @throws Exception If there is any error.
414       */
415      private void processCommitEntry(SensorShell shell, String author, long timestamp, 
416          Date commitTime, int revision, PerforceChangeListData.PerforceFileData fileData)
417        throws Exception {
418        if (shell != null) {
419          String file = this.fileNamePrefix == null ? "" : this.fileNamePrefix;
420          file += fileData.getFileName();
421    
422          Map<String, String> pMap = new HashMap<String, String>();
423          String timestampString = Tstamp.makeTimestamp(timestamp).toString();
424          pMap.put("SensorDataType", "Commit");
425          pMap.put("Resource", file);
426          pMap.put("Tool", "Perforce");
427          pMap.put("Timestamp", timestampString);
428          pMap.put("Runtime", Tstamp.makeTimestamp(commitTime.getTime()).toString());
429          pMap.put("totalLines", String.valueOf(fileData.getTotalLines()));
430          pMap.put("linesAdded", String.valueOf(fileData.getLinesAdded()));
431          pMap.put("linesDeleted", String.valueOf(fileData.getLinesDeleted()));
432          pMap.put("linesModified", String.valueOf(fileData.getLinesModified()));
433          shell.add(pMap);
434          if (this.isVerbose) {
435            System.out.printf("Sending Perforce Commit: Timestamp: %s Resource: %s User: %s%n", 
436                timestampString, file, shell.getProperties().getSensorBaseUser());
437          }
438        }
439      }
440    
441    }