001    package org.hackystat.sensor.ant.perforce;
002    
003    import java.util.ArrayList;
004    import java.util.Date;
005    import java.util.List;
006    import java.util.Vector;
007    
008    import com.perforce.api.Change;
009    import com.perforce.api.Client;
010    import com.perforce.api.Env;
011    import com.perforce.api.FileEntry;
012    import com.perforce.api.P4Process;
013    import com.perforce.api.Utils;
014    
015    /**
016     * Provides the interface to Perforce for the sensor. This code accomplishes the following:
017     * <ul>
018     * <li> Defines a new Client with a view that maps to the Perforce depot directory of interest.  
019     * <li> Obtains the set of ChangeLists for a specified date interval.
020     * <li> Finds the set of files committed in each change list. 
021     * <li> Performs a diff on each file to get a count of lines added, modified, and changed.
022     * </ul>
023     * 
024     * Note that you must create a P4Environment instance and initialize it properly before 
025     * invoking the PerforceCommitProcessor.  
026     * 
027     * See the main() method for example usage of this class. 
028     * 
029     * @author Philip Johnson
030     */
031    public class PerforceCommitProcessor {
032    
033      /** The Perforce depotPath associated with this processor. */
034      private String depotPath;
035      /** The Perforce Java API Env instance initialized in setupEnvironment. */
036      private Env env;
037      /** The Client workspace that will be created based upon the depotPath. */
038      private Client client;
039      /** The set of changelists and associated information we will build in this processor. */  
040      private List<PerforceChangeListData> changeListDataList = new ArrayList<PerforceChangeListData>();
041      
042      /** Controls whether the -dw option is passed to diff2. */
043      private boolean ignoreWhitespace = false;
044      
045      /** Disable the default public no-arg constructor. */
046      @SuppressWarnings("unused")
047      private PerforceCommitProcessor () {
048        // Disable default constructor.
049      }
050      
051      /**
052       * Instantiates a PerforceCommitProcessor with the passed P4Environment instance and depotPath. 
053       * @param p4Environment The p4Environment. 
054       * @param depotPath The depot path this processor will work on. 
055       * @throws Exception If problems occur instantiating this environment. 
056       */
057      public PerforceCommitProcessor(P4Environment p4Environment, String depotPath) throws Exception {
058        this.env = p4Environment.getEnv();
059        this.depotPath = depotPath;
060        this.client = createClient(env, depotPath);
061        this.env.setClient(this.client.getName());
062      }
063      
064    
065      /**
066       * Creates a Perforce Client that maps the depotPath to a local workspace.
067       * This Client has a unique name which enables concurrent execution of this sensor. 
068       * It will be deleted during cleanup.
069       * @param env The Env instance to be used to create this client.
070       * @param depotPath The depotPath this Client will map.   
071       * @return The newly created Client.  
072       * @throws Exception if problems occur. 
073       */
074      private Client createClient(Env env, String depotPath) throws Exception {
075        String hackystatSensorClientName = "hackystat-sensor-" + (new Date()).getTime();
076        Client client = new Client(env, hackystatSensorClientName);
077        client.setRoot(System.getProperty("user.home") + "/perforcesensorsketch");
078        client.addView(depotPath, "//" + hackystatSensorClientName + "/...");
079        client.commit();
080        return client;
081      }
082    
083      
084      /**
085       * Processes any ChangeLists that were submitted between startData and endDate to the depotPath.
086       * @param startDate The start date, in YYYY/MM/DD format. 
087       * @param endDate The end date, in YYYY/MM/DD format. 
088       * @throws Exception If problems occur. 
089       */
090      @SuppressWarnings("unchecked") // Vector in Perforce Java API is not generic.
091      public void processChangeLists(String startDate, String endDate) throws Exception {
092        int maximumChanges = 1000;
093        boolean useIntegrations = true;
094        String startTime = startDate + " 00:00:00";
095        String endTime = endDate + " 23:59:59";
096        Change[] changes = Change.getChanges(env, depotPath, maximumChanges, startTime, endTime,
097            useIntegrations, null);
098        for (Change changelist : changes) {
099          String owner = changelist.getUser().getId();
100          PerforceChangeListData changeListData = new PerforceChangeListData(owner, changelist
101              .getNumber(), changelist.getModtimeString());
102          changelist.sync();
103          Vector<FileEntry> files = changelist.getFileEntries();
104          for (FileEntry fileEntry : files) {
105            //fileEntry.sync(); // not sure if this is needed. Maybe changelist.sync() is good enough.
106            
107            // Changelists can contain files not in the user-specified depotPath, so only process 
108            // files in the changelist that match the depotPath. 
109            if (Utils.wildPathMatch(this.depotPath, fileEntry.getDepotPath())) {
110              // Set up defaults for size info for binary files.
111              Integer[] lineInfo = { 0, 0, 0 };
112              int totalLoc = 0;
113              // Calculate real values for text files.
114              if (isTextFile(fileEntry.getDepotPath(), changelist.getNumber())) {
115                lineInfo = getFileChangeInfo(fileEntry);
116                totalLoc = getFileSize(changelist.getNumber(), fileEntry.getDepotPath());
117              }
118              changeListData.addFileData(fileEntry.getDepotPath(), lineInfo[0], lineInfo[1],
119                  lineInfo[2], totalLoc);
120            }
121          }
122          this.changeListDataList.add(changeListData);
123        }
124      }
125      
126      /**
127       * Retrieve the list of PerforceChangeListData instances associated with this instance. 
128       * @return The list of PerforceChangeListData instances.  
129       */
130      public List<PerforceChangeListData> getChangeListDataList() {
131        return this.changeListDataList;
132      }
133      
134      /**
135       * Controls whether the diff2 command will be passed the -dw option so that whitespace changes
136       * in the file are ignored. 
137       * @param ignoreWhitespace True if whitespace changes in the file should be ignored. 
138       */
139      public void setIgnoreWhitespace(boolean ignoreWhitespace) {
140        this.ignoreWhitespace = ignoreWhitespace;
141      }
142    
143      /**
144       * Finds out the lines added, deleted, and changed for the passed file.
145       * 
146       * @param entry The FileEntry from the change list.
147       * @return A three-tuple containing the lines added, deleted, and modified for this file.
148       * @throws Exception If problems occur.   
149       */
150      private Integer[] getFileChangeInfo(FileEntry entry) throws Exception {
151        entry.sync();
152        String depotPath = entry.getDepotPath();
153        Integer[] ints = {0, 0, 0};
154        int revision = entry.getHeadRev();
155        String difference = runDiff2Command(depotPath, revision);
156        ints = processDiff2Output(difference);
157        return ints;
158      }
159    
160      /**
161       * Invokes the p4 diff2 command to obtain summary information about the differences between
162       * the two files.  Returns the output of the command.
163       * @param file The file to be diffed.
164       * @param revision The original revision number. Will be diffed against the prior revision number.
165       * @return The output from running the command. 
166       * @throws Exception If problems occur.
167       */
168      private String runDiff2Command(String file, int revision) throws Exception {
169        int priorRevision = revision - 1;
170        List<String> cmd = new ArrayList<String>();
171        cmd.add("p4");
172        cmd.add("diff2");
173        cmd.add("-ds");
174        if (this.ignoreWhitespace) {
175          cmd.add("-dw");
176        }
177        cmd.add(file + "#" + priorRevision);
178        cmd.add(file + "#" + revision);
179        String[] args = cmd.toArray(new String[cmd.size()]); 
180        return runP4Command(args);
181      }
182      
183      
184      /**
185       * Invokes the p4 program with the specified arguments, and returns the output as a string.
186       * @param cmd The command to be invoked. 
187       * @return The output from the P4 program. 
188       * @throws Exception If problems occur. 
189       */
190      private String runP4Command (String[] cmd) throws Exception {
191        String l;
192        StringBuffer sb = new StringBuffer();
193        P4Process p = new P4Process(this.env);
194        p.exec(cmd);
195        while (null != (l = p.readLine())) {
196          sb.append(l);
197          sb.append('\n');
198        }
199        p.close();
200        return sb.toString();
201      }
202    
203      /**
204       * Calls the p4 program to delete the specified client instance. 
205       * (This should really be part of the official Perforce Java API.)
206       * @param client The client to be deleted. 
207       * @return The string returned by perforce. 
208       * @throws Exception if problems occur. 
209       */
210      private String runDeleteClientCommand(Client client) throws Exception {
211        String[] cmd = { "p4", "client", "-d", client.getName() };
212        return runP4Command(cmd);
213      }
214      
215      /**
216       * Calls the p4 program to get file size in LOC for the specified file. 
217       * (This should really be part of the official Perforce Java API.)
218       * (We also shouldn't have to retrieve the whole darn file just to count the number of lines.)
219       * Only call this if the file is text.
220       * @param changelist The changelist revision we want file size for. 
221       * @param file The file we want stats for. Should be in the form of a depot path.  
222       * @return The number of lines in this file. 
223       * @throws Exception if problems occur. 
224       */
225      private int getFileSize(int changelist, String file) throws Exception {
226        String[] cmd = { "p4", "print", file + "@" + changelist };
227        P4Process p = new P4Process(this.env);
228        p.exec(cmd);
229        int loc = 0;
230        while (null != p.readLine()) {
231          loc++;
232        }
233        p.close();
234        return loc;
235      }
236      
237      /**
238       * Calls the p4 program to get file type for the specified file, and returns true if
239       * the file type is 'text'. 
240       * (This should really be part of the official Perforce Java API.)
241       * @param file The file we want stats for. Should be in the form of a depot path.  
242       * @param changelist The changelist revision we want file size for. 
243       * @return True if the file type is text.
244       * @throws Exception if problems occur. 
245       */
246      private boolean isTextFile(String file, int changelist) throws Exception {
247        String[] cmd = { "p4", "files", file + "@" + changelist };
248        String fileInfo = runP4Command(cmd);
249        return fileInfo.contains("(text)");
250      }
251      
252      
253      
254      /**
255       * Takes the diff2 command output, and parses it to produce an array of three integers: the
256       * lines added, the lines deleted, and the lines changed. 
257       * 
258       * Typical diff2 output might look like this:
259       * <pre>
260       * Diffing file: //depot/project/Foo.java
261       * ==== //depot/project/Foo.java#1 (text) - //depot/project/Foo.java#2 (text) ==== content
262       * add 2 chunks 6 lines
263       * deleted 0 chunks 0 lines
264       * changed 3 chunks 8 / 10 lines
265       * </pre>
266       * 
267       * Note that the diff2 command returns two numbers for "changed" regions of a file.  The first
268       * number is the number of lines deleted from the changed region, while the second number
269       * is the number of lines added to the changed region.   We will take the smaller of the 
270       * two numbers to be the "changed" count, and then the difference between the two numbers
271       * will be added to the "add" count.  (Recommended by Greg Bylenok.)
272       *   
273       * @param output The diff2 command output
274       * @return An array of three integers containing added, deleted, and changed lines. 
275       * @throws Exception If problems occur. 
276       */
277      private Integer[] processDiff2Output(String output) throws Exception {
278        Integer[] ints = { 0, 0, 0 };
279        String[] lines = output.split("\\n");
280        for (String line : lines) {
281          String[] tokens = line.split("\\s");
282          String changeType = tokens[0];
283          if ("add".equals(changeType)) {
284            ints[0] += Integer.valueOf(tokens[3]);
285          }
286          if ("deleted".equals(changeType)) {
287            ints[1] += Integer.valueOf(tokens[3]);
288          }
289          if ("changed".equals(changeType)) {
290            int changedLinesDeleted = Integer.valueOf(tokens[3]);
291            int changedLinesAdded = Integer.valueOf(tokens[5]);
292            int min = Math.min(changedLinesAdded, changedLinesDeleted);
293            int diff = Math.abs(changedLinesDeleted - changedLinesAdded);
294            ints[2] += min;
295            ints[0] += diff;
296          }
297        }
298        return ints;
299      }
300      
301      /**
302       * This method should be invoked at the end of the sensor run, and will delete the client
303       * created for this task as well as invoke the Perforce library cleanUp() method.
304       * @throws Exception If problems occur. 
305       */
306      public void cleanup() throws Exception {
307        this.runDeleteClientCommand(this.client);
308        Utils.cleanUp();
309      }
310    
311      /**
312       * Exercises the methods in this class manually.  Useful as a way to check that you have
313       * configured your p4 environment correctly if you are having problems with the sensor. 
314       * Supply arguments in the following order. Examples given in parentheses:
315       * <ul>
316       * <li> port ("public.perforce.com:1666")
317       * <li> user ("philip_johnson")
318       * <li> password ("foo")
319       * <li> depotPath ("//guest/philip_johnson/...")
320       * <li> startDate ("2008/07/14")
321       * <li> endDate ("2008/07/15")
322       * </ul>
323       * @param args Arguments are: port, user, password, depotPath, startDate, endDate.
324       * @throws Exception If problems occur. 
325       */
326      public static void main(String[] args) throws Exception {
327        System.out.printf("Starting PerforceCommitProcessor. %nSetting up environment...");
328        P4Environment p4Env = new P4Environment();
329        p4Env.setP4Port(args[0]);
330        p4Env.setP4User(args[1]);
331        p4Env.setP4Password(args[2]);
332        p4Env.setVerbose(false); // could set this to true for lots of debugging output. 
333        PerforceCommitProcessor processor = new PerforceCommitProcessor(p4Env, args[3]);
334        System.out.printf("done. %nNow retrieving change lists...");
335        processor.processChangeLists(args[4], args[5]);
336        System.out.printf("found %d changelists. %n", processor.changeListDataList.size());
337        for (PerforceChangeListData data : processor.getChangeListDataList()) {
338          System.out.println(data);
339          for (PerforceChangeListData.PerforceFileData fileData : data.getFileData()) {
340            System.out.printf("Size of %s is %d%n", fileData.getFileName(), 
341                processor.getFileSize(data.getId(), fileData.getFileName()));
342          }
343        }
344        // Always make sure you call cleanup() at the end. 
345        processor.cleanup();
346        System.out.printf("Finished PerforceCommitProcessor.");
347      }
348      
349    }