Added a new entity to hold device type stats, as querying all objects from AppEngine takes too long

appengine/src/com/google/android/chrometophone/server/DeviceInfo.java

@@ -19,12 +19,10 @@ package com.google.android.chrometophone.server;
 import com.google.appengine.api.datastore.Key;
 
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
 import java.util.Date;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
-import java.util.logging.Level;
-import java.util.logging.Logger;
 
 import javax.jdo.PersistenceManager;
 import javax.jdo.Query;
@@ -43,12 +41,13 @@ import javax.jdo.annotations.PrimaryKey;
  */
 @PersistenceCapable(identityType = IdentityType.APPLICATION)
 public class DeviceInfo {
-    private static final Logger log =
-      Logger.getLogger(DeviceInfo.class.getName());
   
     public static final String TYPE_AC2DM = "ac2dm";
     public static final String TYPE_CHROME = "chrome";
     public static final String TYPE_GCM = "gcm";
+    
+    public static final List<String> SUPPORTED_TYPES = Collections
+        .unmodifiableList(Arrays.asList(TYPE_AC2DM, TYPE_CHROME, TYPE_GCM));
 
     /**
      * User-email # device-id
@@ -89,6 +88,10 @@ public class DeviceInfo {
     @Persistent
     private Date registrationTimestamp;
 
+    /**
+     * Debug is not used anymore, but since the datastore already has it, it is
+     * now used to flag whether this object has been added to the stats or not.
+     */
     @Persistent
     private Boolean debug;
 
@@ -124,8 +127,8 @@ public class DeviceInfo {
         return (debug != null ? debug.booleanValue() : false);
     }
 
-    public void setDebug(boolean debug) {
+    public void setDebug(Boolean debug) {
-        this.debug = new Boolean(debug);
+        this.debug = debug;
     }
 
     public void setType(String type) {
@@ -169,27 +172,5 @@ public class DeviceInfo {
         query.closeAll();
         return result;
     }
-    
-    /**
-     * Helper function - get number of devices registered by type.
-     */
-    public static Map<String, Integer> getDevicesUsage(PersistenceManager pm) {
-        Map<String, Integer> result = new HashMap<String, Integer>();
-        Query query = pm.newQuery(DeviceInfo.class);
-        addStats(query, result, TYPE_AC2DM);
-        addStats(query, result, TYPE_GCM);
-        addStats(query, result, TYPE_CHROME);
-        query.closeAll();
-        return result;
-    }
 
-    private static void addStats(Query query, Map<String, Integer> result, String type) {
-      query.setResult("count(this)");
-      query.setFilter("type == '" + type + "'");
-      Integer total = (Integer) query.execute();
-      log.log(Level.INFO, "Number of records of type {0}: {1}",
-          new Object[] {type, total});
-      result.put(type, total);
-    }
-    
 }

appengine/src/com/google/android/chrometophone/server/DeviceStats.java

@@ -0,0 +1,179 @@
+/*
+ * Copyright 2012 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.android.chrometophone.server;
+
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import javax.jdo.JDOObjectNotFoundException;
+import javax.jdo.PersistenceManager;
+import javax.jdo.Query;
+import javax.jdo.annotations.IdGeneratorStrategy;
+import javax.jdo.annotations.IdentityType;
+import javax.jdo.annotations.PersistenceCapable;
+import javax.jdo.annotations.Persistent;
+import javax.jdo.annotations.PrimaryKey;
+
+/**
+ * Statistics about a device type.
+ */
+@PersistenceCapable(identityType = IdentityType.APPLICATION)
+public class DeviceStats {
+
+    private static final Logger log =
+      Logger.getLogger(DeviceStats.class.getName());
+
+    /**
+     * Device type, as defined on {@link DeviceInfo}.
+     */
+    @Persistent(valueStrategy = IdGeneratorStrategy.IDENTITY)
+    @PrimaryKey
+    private String type;
+
+    /**
+     * Current number of devices using this type.
+     */
+    @Persistent
+    private int total;
+
+    /**
+     * Total number of devices added for this type.
+     */
+    @Persistent
+    private int added;
+
+    /**
+     * Current number of devices using this type.
+     */
+    @Persistent
+    private int deleted;
+
+    /**
+     * Current number of devices converted to this type
+     */
+    @Persistent
+    private int converted;
+    
+    private DeviceStats(String type) {
+        this.type = type;
+        this.total = this.added = this.deleted = 0;
+    }
+
+    public String getType() {
+      return type;
+    }
+
+    public int getTotal() {
+      return total;
+    }
+
+    public int getAdded() {
+      return added;
+    }
+
+    public int getDeleted() {
+      return deleted;
+    }
+
+    public int getConverted() {
+      return converted;
+    }
+
+    @Override
+    public String toString() {
+      return String.format("DeviceStats[%s]: total=%d, added=%d, deleted=%d, converted=%d",
+          type, total, added, deleted, converted);
+    }
+
+    /**
+     * Queries the stats for a given type.
+     */
+    public static DeviceStats getStats(PersistenceManager pm, String type) {
+        Query query = pm.newQuery(DeviceStats.class);
+        DeviceStats stats = null;
+        Object key = pm.newObjectIdInstance(DeviceStats.class, type);
+        try {
+          stats = (DeviceStats) pm.getObjectById(key);
+          log.log(Level.INFO, "getStats(): {0}", stats);
+        } catch (JDOObjectNotFoundException e) {
+          log.log(Level.INFO, "getStats() not found for type {0}", type);
+        } finally {
+          query.closeAll();
+        }
+        return stats;
+    }
+
+    /**
+     * Adds one device by a given type.
+     */
+    static DeviceStats addDevice(PersistenceManager pm, String type) {
+        DeviceStats stats = getStats(pm, type);
+        stats.total++;
+        stats.added++;
+        return stats;
+    }
+
+    /**
+     * Removes one device by a given type.
+     */
+    static DeviceStats removeDevice(PersistenceManager pm, String type) {
+        DeviceStats stats = getStats(pm, type);
+        stats.total--;
+        stats.deleted++;
+        return stats;
+    }
+
+    /**
+     * Converts one device by a given type.
+     */
+    static DeviceStats convertsDevice(PersistenceManager pm, String type, int size) {
+        log.log(Level.INFO, "Updating entity of type {0} with {1} conversions",
+            new Object[] {type, size});
+        DeviceStats stats = getStats(pm, type);
+        stats.total += size;
+        stats.converted += size;
+        return stats;
+    }
+ 
+    static List<DeviceStats> getAll(PersistenceManager pm) {
+      Query query = pm.newQuery(DeviceStats.class);
+      @SuppressWarnings("unchecked")
+      List<DeviceStats> allStats = (List<DeviceStats>) query.execute();
+      return allStats;
+    }
+    
+    /**
+     * Create initial objects.
+     */
+    static void init(PersistenceManager pm) {
+       for (String type : DeviceInfo.SUPPORTED_TYPES) {
+         add(pm, type);
+       }
+    }
+
+    private static void add(PersistenceManager pm, String type) {
+        DeviceStats stats = getStats(pm, type);
+        if (stats == null) {
+          stats = new DeviceStats(type);
+          log.log(Level.INFO, "Creating {0} entity for type {1}",
+              new Object[] {DeviceStats.class.getSimpleName(), type});
+          pm.makePersistent(stats);
+        }
+    }
+
+}

appengine/src/com/google/android/chrometophone/server/RegisterServlet.java

@@ -16,6 +16,14 @@
 
 package com.google.android.chrometophone.server;
 
+import com.google.android.c2dm.server.C2DMessaging;
+import com.google.appengine.api.channel.ChannelServiceFactory;
+import com.google.appengine.api.datastore.Key;
+import com.google.appengine.api.datastore.KeyFactory;
+import com.google.appengine.labs.repackaged.org.json.JSONArray;
+import com.google.appengine.labs.repackaged.org.json.JSONException;
+import com.google.appengine.labs.repackaged.org.json.JSONObject;
+
 import java.io.IOException;
 import java.io.PrintWriter;
 import java.util.Date;
@@ -29,14 +37,6 @@ import javax.servlet.http.HttpServlet;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
-import com.google.android.c2dm.server.C2DMessaging;
-import com.google.appengine.api.channel.ChannelServiceFactory;
-import com.google.appengine.api.datastore.Key;
-import com.google.appengine.api.datastore.KeyFactory;
-import com.google.appengine.labs.repackaged.org.json.JSONArray;
-import com.google.appengine.labs.repackaged.org.json.JSONException;
-import com.google.appengine.labs.repackaged.org.json.JSONObject;
-
 @SuppressWarnings("serial")
 public class RegisterServlet extends HttpServlet {
     private static final Logger log =
@@ -119,7 +119,6 @@ public class RegisterServlet extends HttpServlet {
         // Context-shared PMF.
         PersistenceManager pm =
             C2DMessaging.getPMF(getServletContext()).getPersistenceManager();
-
         try {
             List<DeviceInfo> registrations = reqInfo.devices;
 
@@ -130,7 +129,7 @@ public class RegisterServlet extends HttpServlet {
                 // unused registrations
                 DeviceInfo oldest = registrations.get(0);
                 if (oldest.getRegistrationTimestamp() == null) {
-                    reqInfo.deleteRegistration(oldest.getDeviceRegistrationID());
+                    reqInfo.deleteRegistration(oldest.getDeviceRegistrationID(), deviceType);
                 } else {
                     long oldestTime = oldest.getRegistrationTimestamp().getTime();
                     for (int i = 1; i < registrations.size(); i++) {
@@ -140,7 +139,7 @@ public class RegisterServlet extends HttpServlet {
                             oldestTime = oldest.getRegistrationTimestamp().getTime();
                         }
                     }
-                    reqInfo.deleteRegistration(oldest.getDeviceRegistrationID());
+                    reqInfo.deleteRegistration(oldest.getDeviceRegistrationID(), deviceType);
                 }
             }
 
@@ -167,6 +166,7 @@ public class RegisterServlet extends HttpServlet {
             // TODO: only need to write if something changed, for chrome nothing
             // changes, we just create a new channel
             pm.makePersistent(device);
+            DeviceStats.addDevice(pm, deviceType);
             log.log(Level.INFO, "Registered device " + reqInfo.userName + " " +
                     deviceType);
 

appengine/src/com/google/android/chrometophone/server/RequestInfo.java

@@ -212,7 +212,7 @@ public class RequestInfo {
     // We need to iterate again - can be avoided with a query.
     // delete will fail if the pm is different than the one used to
     // load the object - we must close the object when we're done
-    public void deleteRegistration(String regId) {
+    public void deleteRegistration(String regId, String type) {
         if (ctx == null) {
             return;
         }
@@ -224,6 +224,7 @@ public class RequestInfo {
                 DeviceInfo deviceInfo = registrations.get(i);
                 if (deviceInfo.getDeviceRegistrationID().equals(regId)) {
                     pm.deletePersistent(deviceInfo);
+                    DeviceStats.removeDevice(pm, type);
                     // Keep looping in case of duplicates
                 }
             }

appengine/src/com/google/android/chrometophone/server/SendServlet.java

@@ -21,9 +21,7 @@ import com.google.appengine.api.channel.ChannelMessage;
 import com.google.appengine.api.channel.ChannelServiceFactory;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Iterator;
-import java.util.List;
 import java.util.logging.Logger;
 
 import javax.servlet.http.HttpServlet;
@@ -133,7 +131,8 @@ public class SendServlet extends HttpServlet {
                 if (ex != null) {
                     if ("InvalidRegistration".equals(ex.getMessage())) {
                       // Prune device, it no longer works
-                      reqInfo.deleteRegistration(deviceInfo.getDeviceRegistrationID());
+                      reqInfo.deleteRegistration(deviceInfo.getDeviceRegistrationID(),
+                          deviceInfo.getType());
                       iterator.remove();
                       ac2dmCnt--;
                     } else {

appengine/src/com/google/android/chrometophone/server/StatsConversionServlet.java

@@ -0,0 +1,109 @@
+/*
+ * Copyright 2012 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.chrometophone.server;
+
+import com.google.android.c2dm.server.C2DMessaging;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import javax.jdo.PersistenceManager;
+import javax.jdo.Query;
+import javax.servlet.ServletContext;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+/**
+ * Adds devices whose debug field is not set to the stats, and update the field.
+ */
+public class StatsConversionServlet extends HttpServlet {
+  
+    private static final int MAX_ROWS = 500;
+    
+    @Override
+    public void doGet(HttpServletRequest req, HttpServletResponse resp)
+        throws IOException{
+      ServletContext ctx = getServletContext();
+      List<DeviceInfo> devices = null;
+      PersistenceManager pm = C2DMessaging.getPMF(ctx).getPersistenceManager();
+      Map<String, Integer> conversionsByType = new HashMap<String, Integer>();
+      try {
+        resp.setContentType("text/html");
+        PrintWriter out = resp.getWriter();
+        out.println("<html><body>");
+        out.println("<head>");
+        out.println("<title>Device stats conversion</title>");
+        out.println("</head>");
+        out.println("<body>");
+        int total = 0;
+        do {
+          Query query = pm.newQuery(DeviceInfo.class);
+          query.setFilter("debug == null");
+          query.setRange(0, MAX_ROWS);
+          @SuppressWarnings("unchecked")
+          List<DeviceInfo> uncastDevices = (List<DeviceInfo>) query.execute();
+          devices = uncastDevices;
+          if (devices.isEmpty()) {
+            out.println("<p>No devices need conversion</p>");
+          } else {
+            int size = devices.size();
+            total += size;
+            out.println("<p>Converting " + size + " devices...");
+            out.flush();
+            for (DeviceInfo device : devices) {
+              device.setDebug(true);
+              pm.currentTransaction().begin();
+              pm.makePersistent(device);
+              pm.currentTransaction().commit();
+              String type = device.getType();
+              Integer converted = conversionsByType.get(type);
+              if (converted == null) {
+                converted = 1;
+              } else {
+                converted ++;
+              }
+              conversionsByType.put(type, converted);
+            }
+          }
+          query.closeAll();
+          flushStats(pm, conversionsByType);
+          out.println(total + " converted so far.</p>");
+        } while (devices != null && ! devices.isEmpty());
+        out.println("</body></html>");
+      } finally {
+        pm.close();
+      }
+      resp.setStatus(HttpServletResponse.SC_OK);
+    }
+
+    private void flushStats(PersistenceManager pm, Map<String, Integer> conversionsByType) {
+      for (Entry<String, Integer> entry : conversionsByType.entrySet()) {
+        String type = entry.getKey();
+        int size = entry.getValue();
+        DeviceStats stats = DeviceStats.convertsDevice(pm, type, size);
+        pm.currentTransaction().begin();
+        pm.makePersistent(stats);
+        pm.currentTransaction().commit();
+      }
+      conversionsByType.clear();
+    }
+
+}

appengine/src/com/google/android/chrometophone/server/StatsServlet.java

@@ -19,8 +19,7 @@ import com.google.android.c2dm.server.C2DMessaging;
 
 import java.io.IOException;
 import java.io.PrintWriter;
-import java.util.Map;
+import java.util.List;
-import java.util.Map.Entry;
 
 import javax.jdo.PersistenceManager;
 import javax.servlet.ServletContext;
@@ -39,45 +38,59 @@ public class StatsServlet extends HttpServlet {
     public void doGet(HttpServletRequest req, HttpServletResponse resp)
         throws IOException{
       ServletContext ctx = getServletContext();
+      List<DeviceStats> allStats;
       PersistenceManager pm = C2DMessaging.getPMF(ctx).getPersistenceManager();
-      Map<String, Integer> stats = DeviceInfo.getDevicesUsage(pm);
+      try {
-      pm.close();
+        DeviceStats.init(pm);
-
+        allStats = DeviceStats.getAll(pm);
-      resp.setContentType("text/html");
+        resp.setContentType("text/html");
-      PrintWriter out = resp.getWriter();
+        PrintWriter out = resp.getWriter();
-      out.println("<html><body>");
+        out.println("<html><body>");
-      out.println("<head>");
+        out.println("<head>");
-      out.println("<title>Device stats</title>");
+        out.println("<title>Device stats</title>");
-      out.println("</head>");
+        out.println("</head>");
-      out.println("<body>");
+        out.println("<body>");
-      out.println("<h3>Device stats</h3>");
+        out.println("<h3>Device stats</h3>");
-      if (stats.isEmpty()) {
+        if (allStats.isEmpty()) {
-        out.println("<p>No devices registered yet!</p>");
+          out.println("<p>No devices registered yet!</p>");
-      } else {
+        } else {
-        int total = 0;
+          int total = 0;
-        for (Integer count : stats.values()) {
+          int added = 0;
-          total += count;
+          int deleted = 0;
-        }
+          int converted = 0;
-
+          for (DeviceStats stats: allStats) {
-        out.println("<table cellspacing='2' cellpadding='2'><tr><th>Type</th>" +
+            total += stats.getTotal();
-        		"<th>Count</th><th>Share</th></tr>");
+            added += stats.getAdded();
-        for (Entry<String, Integer> entry : stats.entrySet()) {
+            deleted += stats.getDeleted();
-          String type = entry.getKey();
+            converted += stats.getConverted();
-          int count = entry.getValue();
+          }
-          float share = (100*count) / total; 
+          out.println("<table cellspacing='2' cellpadding='2'><tr><th>Type</th>" +
+          		"<th>Added</th><th>Deleted</th><th>Converted</th><th>Total</th><th>Share</th></tr>");
+          for (DeviceStats stats: allStats) {
+            int count = stats.getTotal();
+            float share = (total == 0) ? 0 : (100 * count) / total; 
+            out.println("<tr>" +
+                  "<td align='right'>" + stats.getType() + "</td>" +
+                  "<td align='right'>" + stats.getAdded() + "</td>" +
+                  "<td align='right'>" + stats.getDeleted() + "</td>" +
+                  "<td align='right'>" + stats.getConverted() + "</td>" +
+                  "<td align='right'>" + count + "</td>" +
+                  "<td align='right'>" + share + "%</td></tr>");
+          }
           out.println("<tr>" +
-                "<td align='right'>" + type + "</td>" +
+                "<td align='right'>Total</td>" +
-                "<td align='right'>" + count + "</td>" +
+                "<td align='right'>" + added + "</td>" +
-                "<td align='right'>" + share + "%</td></tr>");
+                "<td align='right'>" + deleted + "</td>" +
+                "<td align='right'>" + converted + "</td>" +
+                "<td align='right'>" + total + "</td>" +
+                "<td align='right'>100.0%</td></tr>");
+          out.println("</table>");
         }
-        out.println("<tr>" +
+        
-              "<td align='right'>Total</td>" +
+        out.println("</body></html>");
-              "<td align='right'>" + total+ "</td>" +
+      } finally {
-              "<td align='right'>100.0%</td></tr>");
+        pm.close();
-        out.println("</table>");
       }
-      
-      out.println("</body></html>");
       resp.setStatus(HttpServletResponse.SC_OK);
     }
 }

appengine/src/com/google/android/chrometophone/server/UnregisterServlet.java

@@ -43,7 +43,13 @@ public class UnregisterServlet extends HttpServlet {
             return;
         }
 
-        reqInfo.deleteRegistration(reqInfo.deviceRegistrationID);
+        // TODO: make sure new app passes deviceType
+        String deviceType = reqInfo.getParameter("deviceType");
+        if (deviceType == null) {
+            deviceType = "ac2dm";
+        }
+
+        reqInfo.deleteRegistration(reqInfo.deviceRegistrationID, deviceType);
         resp.getWriter().println(OK_STATUS);
     }
 }

appengine/war/WEB-INF/appengine-web.xml

@@ -16,7 +16,7 @@
 -->
 <appengine-web-app xmlns="http://appengine.google.com/ns/1.0">
     <application>chrometophone</application>
-    <version>11</version>
+    <version>dev</version>
     <threadsafe>true</threadsafe>
     <system-properties>
         <property name="java.util.logging.config.file" value="WEB-INF/logging.properties"/>

appengine/war/WEB-INF/web.xml

@@ -74,6 +74,12 @@
     </servlet-class>
   </servlet>
 
+  <servlet>
+    <servlet-name>StatsConversionServlet</servlet-name>
+    <servlet-class>com.google.android.chrometophone.server.StatsConversionServlet
+    </servlet-class>
+  </servlet>
+
 	<servlet-mapping>
 		<servlet-name>RegisterServlet</servlet-name>
 		<url-pattern>/register</url-pattern>
@@ -119,6 +125,11 @@
         <url-pattern>/admin/stats</url-pattern>
     </servlet-mapping>
     
+    <servlet-mapping>
+        <servlet-name>StatsConversionServlet</servlet-name>
+        <url-pattern>/admin/convertStats</url-pattern>
+    </servlet-mapping>
+
 	<servlet-mapping>
 		<servlet-name>dataMessagingServlet</servlet-name>
 		<url-pattern>/tasks/c2dm</url-pattern>