From 87f64215f95d20ea7896b2fe231bba794aea02ab Mon Sep 17 00:00:00 2001 From: felipeal Date: Tue, 31 Jul 2012 02:30:16 +0000 Subject: [PATCH] Added a new entity to hold device type stats, as querying all objects from AppEngine takes too long --- .../chrometophone/server/DeviceInfo.java | 41 ++-- .../chrometophone/server/DeviceStats.java | 179 ++++++++++++++++++ .../chrometophone/server/RegisterServlet.java | 22 +-- .../chrometophone/server/RequestInfo.java | 3 +- .../chrometophone/server/SendServlet.java | 5 +- .../server/StatsConversionServlet.java | 109 +++++++++++ .../chrometophone/server/StatsServlet.java | 87 +++++---- .../server/UnregisterServlet.java | 8 +- appengine/war/WEB-INF/appengine-web.xml | 2 +- appengine/war/WEB-INF/web.xml | 11 ++ 10 files changed, 383 insertions(+), 84 deletions(-) create mode 100644 appengine/src/com/google/android/chrometophone/server/DeviceStats.java create mode 100644 appengine/src/com/google/android/chrometophone/server/StatsConversionServlet.java diff --git a/appengine/src/com/google/android/chrometophone/server/DeviceInfo.java b/appengine/src/com/google/android/chrometophone/server/DeviceInfo.java index 7a3ad09..3b03b18 100644 --- a/appengine/src/com/google/android/chrometophone/server/DeviceInfo.java +++ b/appengine/src/com/google/android/chrometophone/server/DeviceInfo.java @@ -19,12 +19,10 @@ package com.google.android.chrometophone.server; import com.google.appengine.api.datastore.Key; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.Date; -import java.util.HashMap; import java.util.List; -import java.util.Map; -import java.util.logging.Level; -import java.util.logging.Logger; import javax.jdo.PersistenceManager; import javax.jdo.Query; @@ -43,12 +41,13 @@ import javax.jdo.annotations.PrimaryKey; */ @PersistenceCapable(identityType = IdentityType.APPLICATION) public class DeviceInfo { - private static final Logger log = - Logger.getLogger(DeviceInfo.class.getName()); public static final String TYPE_AC2DM = "ac2dm"; public static final String TYPE_CHROME = "chrome"; public static final String TYPE_GCM = "gcm"; + + public static final List SUPPORTED_TYPES = Collections + .unmodifiableList(Arrays.asList(TYPE_AC2DM, TYPE_CHROME, TYPE_GCM)); /** * User-email # device-id @@ -89,6 +88,10 @@ public class DeviceInfo { @Persistent private Date registrationTimestamp; + /** + * Debug is not used anymore, but since the datastore already has it, it is + * now used to flag whether this object has been added to the stats or not. + */ @Persistent private Boolean debug; @@ -124,8 +127,8 @@ public class DeviceInfo { return (debug != null ? debug.booleanValue() : false); } - public void setDebug(boolean debug) { - this.debug = new Boolean(debug); + public void setDebug(Boolean debug) { + this.debug = debug; } public void setType(String type) { @@ -169,27 +172,5 @@ public class DeviceInfo { query.closeAll(); return result; } - - /** - * Helper function - get number of devices registered by type. - */ - public static Map getDevicesUsage(PersistenceManager pm) { - Map result = new HashMap(); - Query query = pm.newQuery(DeviceInfo.class); - addStats(query, result, TYPE_AC2DM); - addStats(query, result, TYPE_GCM); - addStats(query, result, TYPE_CHROME); - query.closeAll(); - return result; - } - private static void addStats(Query query, Map result, String type) { - query.setResult("count(this)"); - query.setFilter("type == '" + type + "'"); - Integer total = (Integer) query.execute(); - log.log(Level.INFO, "Number of records of type {0}: {1}", - new Object[] {type, total}); - result.put(type, total); - } - } diff --git a/appengine/src/com/google/android/chrometophone/server/DeviceStats.java b/appengine/src/com/google/android/chrometophone/server/DeviceStats.java new file mode 100644 index 0000000..1e89577 --- /dev/null +++ b/appengine/src/com/google/android/chrometophone/server/DeviceStats.java @@ -0,0 +1,179 @@ +/* + * Copyright 2012 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.android.chrometophone.server; + +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +import javax.jdo.JDOObjectNotFoundException; +import javax.jdo.PersistenceManager; +import javax.jdo.Query; +import javax.jdo.annotations.IdGeneratorStrategy; +import javax.jdo.annotations.IdentityType; +import javax.jdo.annotations.PersistenceCapable; +import javax.jdo.annotations.Persistent; +import javax.jdo.annotations.PrimaryKey; + +/** + * Statistics about a device type. + */ +@PersistenceCapable(identityType = IdentityType.APPLICATION) +public class DeviceStats { + + private static final Logger log = + Logger.getLogger(DeviceStats.class.getName()); + + /** + * Device type, as defined on {@link DeviceInfo}. + */ + @Persistent(valueStrategy = IdGeneratorStrategy.IDENTITY) + @PrimaryKey + private String type; + + /** + * Current number of devices using this type. + */ + @Persistent + private int total; + + /** + * Total number of devices added for this type. + */ + @Persistent + private int added; + + /** + * Current number of devices using this type. + */ + @Persistent + private int deleted; + + /** + * Current number of devices converted to this type + */ + @Persistent + private int converted; + + private DeviceStats(String type) { + this.type = type; + this.total = this.added = this.deleted = 0; + } + + public String getType() { + return type; + } + + public int getTotal() { + return total; + } + + public int getAdded() { + return added; + } + + public int getDeleted() { + return deleted; + } + + public int getConverted() { + return converted; + } + + @Override + public String toString() { + return String.format("DeviceStats[%s]: total=%d, added=%d, deleted=%d, converted=%d", + type, total, added, deleted, converted); + } + + /** + * Queries the stats for a given type. + */ + public static DeviceStats getStats(PersistenceManager pm, String type) { + Query query = pm.newQuery(DeviceStats.class); + DeviceStats stats = null; + Object key = pm.newObjectIdInstance(DeviceStats.class, type); + try { + stats = (DeviceStats) pm.getObjectById(key); + log.log(Level.INFO, "getStats(): {0}", stats); + } catch (JDOObjectNotFoundException e) { + log.log(Level.INFO, "getStats() not found for type {0}", type); + } finally { + query.closeAll(); + } + return stats; + } + + /** + * Adds one device by a given type. + */ + static DeviceStats addDevice(PersistenceManager pm, String type) { + DeviceStats stats = getStats(pm, type); + stats.total++; + stats.added++; + return stats; + } + + /** + * Removes one device by a given type. + */ + static DeviceStats removeDevice(PersistenceManager pm, String type) { + DeviceStats stats = getStats(pm, type); + stats.total--; + stats.deleted++; + return stats; + } + + /** + * Converts one device by a given type. + */ + static DeviceStats convertsDevice(PersistenceManager pm, String type, int size) { + log.log(Level.INFO, "Updating entity of type {0} with {1} conversions", + new Object[] {type, size}); + DeviceStats stats = getStats(pm, type); + stats.total += size; + stats.converted += size; + return stats; + } + + static List getAll(PersistenceManager pm) { + Query query = pm.newQuery(DeviceStats.class); + @SuppressWarnings("unchecked") + List allStats = (List) query.execute(); + return allStats; + } + + /** + * Create initial objects. + */ + static void init(PersistenceManager pm) { + for (String type : DeviceInfo.SUPPORTED_TYPES) { + add(pm, type); + } + } + + private static void add(PersistenceManager pm, String type) { + DeviceStats stats = getStats(pm, type); + if (stats == null) { + stats = new DeviceStats(type); + log.log(Level.INFO, "Creating {0} entity for type {1}", + new Object[] {DeviceStats.class.getSimpleName(), type}); + pm.makePersistent(stats); + } + } + +} diff --git a/appengine/src/com/google/android/chrometophone/server/RegisterServlet.java b/appengine/src/com/google/android/chrometophone/server/RegisterServlet.java index 25e77c5..3d11cb9 100644 --- a/appengine/src/com/google/android/chrometophone/server/RegisterServlet.java +++ b/appengine/src/com/google/android/chrometophone/server/RegisterServlet.java @@ -16,6 +16,14 @@ package com.google.android.chrometophone.server; +import com.google.android.c2dm.server.C2DMessaging; +import com.google.appengine.api.channel.ChannelServiceFactory; +import com.google.appengine.api.datastore.Key; +import com.google.appengine.api.datastore.KeyFactory; +import com.google.appengine.labs.repackaged.org.json.JSONArray; +import com.google.appengine.labs.repackaged.org.json.JSONException; +import com.google.appengine.labs.repackaged.org.json.JSONObject; + import java.io.IOException; import java.io.PrintWriter; import java.util.Date; @@ -29,14 +37,6 @@ import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import com.google.android.c2dm.server.C2DMessaging; -import com.google.appengine.api.channel.ChannelServiceFactory; -import com.google.appengine.api.datastore.Key; -import com.google.appengine.api.datastore.KeyFactory; -import com.google.appengine.labs.repackaged.org.json.JSONArray; -import com.google.appengine.labs.repackaged.org.json.JSONException; -import com.google.appengine.labs.repackaged.org.json.JSONObject; - @SuppressWarnings("serial") public class RegisterServlet extends HttpServlet { private static final Logger log = @@ -119,7 +119,6 @@ public class RegisterServlet extends HttpServlet { // Context-shared PMF. PersistenceManager pm = C2DMessaging.getPMF(getServletContext()).getPersistenceManager(); - try { List registrations = reqInfo.devices; @@ -130,7 +129,7 @@ public class RegisterServlet extends HttpServlet { // unused registrations DeviceInfo oldest = registrations.get(0); if (oldest.getRegistrationTimestamp() == null) { - reqInfo.deleteRegistration(oldest.getDeviceRegistrationID()); + reqInfo.deleteRegistration(oldest.getDeviceRegistrationID(), deviceType); } else { long oldestTime = oldest.getRegistrationTimestamp().getTime(); for (int i = 1; i < registrations.size(); i++) { @@ -140,7 +139,7 @@ public class RegisterServlet extends HttpServlet { oldestTime = oldest.getRegistrationTimestamp().getTime(); } } - reqInfo.deleteRegistration(oldest.getDeviceRegistrationID()); + reqInfo.deleteRegistration(oldest.getDeviceRegistrationID(), deviceType); } } @@ -167,6 +166,7 @@ public class RegisterServlet extends HttpServlet { // TODO: only need to write if something changed, for chrome nothing // changes, we just create a new channel pm.makePersistent(device); + DeviceStats.addDevice(pm, deviceType); log.log(Level.INFO, "Registered device " + reqInfo.userName + " " + deviceType); diff --git a/appengine/src/com/google/android/chrometophone/server/RequestInfo.java b/appengine/src/com/google/android/chrometophone/server/RequestInfo.java index 05ac4d6..bc8b8e1 100644 --- a/appengine/src/com/google/android/chrometophone/server/RequestInfo.java +++ b/appengine/src/com/google/android/chrometophone/server/RequestInfo.java @@ -212,7 +212,7 @@ public class RequestInfo { // We need to iterate again - can be avoided with a query. // delete will fail if the pm is different than the one used to // load the object - we must close the object when we're done - public void deleteRegistration(String regId) { + public void deleteRegistration(String regId, String type) { if (ctx == null) { return; } @@ -224,6 +224,7 @@ public class RequestInfo { DeviceInfo deviceInfo = registrations.get(i); if (deviceInfo.getDeviceRegistrationID().equals(regId)) { pm.deletePersistent(deviceInfo); + DeviceStats.removeDevice(pm, type); // Keep looping in case of duplicates } } diff --git a/appengine/src/com/google/android/chrometophone/server/SendServlet.java b/appengine/src/com/google/android/chrometophone/server/SendServlet.java index 3e49df5..547105c 100644 --- a/appengine/src/com/google/android/chrometophone/server/SendServlet.java +++ b/appengine/src/com/google/android/chrometophone/server/SendServlet.java @@ -21,9 +21,7 @@ import com.google.appengine.api.channel.ChannelMessage; import com.google.appengine.api.channel.ChannelServiceFactory; import java.io.IOException; -import java.util.ArrayList; import java.util.Iterator; -import java.util.List; import java.util.logging.Logger; import javax.servlet.http.HttpServlet; @@ -133,7 +131,8 @@ public class SendServlet extends HttpServlet { if (ex != null) { if ("InvalidRegistration".equals(ex.getMessage())) { // Prune device, it no longer works - reqInfo.deleteRegistration(deviceInfo.getDeviceRegistrationID()); + reqInfo.deleteRegistration(deviceInfo.getDeviceRegistrationID(), + deviceInfo.getType()); iterator.remove(); ac2dmCnt--; } else { diff --git a/appengine/src/com/google/android/chrometophone/server/StatsConversionServlet.java b/appengine/src/com/google/android/chrometophone/server/StatsConversionServlet.java new file mode 100644 index 0000000..cf414e9 --- /dev/null +++ b/appengine/src/com/google/android/chrometophone/server/StatsConversionServlet.java @@ -0,0 +1,109 @@ +/* + * Copyright 2012 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.chrometophone.server; + +import com.google.android.c2dm.server.C2DMessaging; + +import java.io.IOException; +import java.io.PrintWriter; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import javax.jdo.PersistenceManager; +import javax.jdo.Query; +import javax.servlet.ServletContext; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +/** + * Adds devices whose debug field is not set to the stats, and update the field. + */ +public class StatsConversionServlet extends HttpServlet { + + private static final int MAX_ROWS = 500; + + @Override + public void doGet(HttpServletRequest req, HttpServletResponse resp) + throws IOException{ + ServletContext ctx = getServletContext(); + List devices = null; + PersistenceManager pm = C2DMessaging.getPMF(ctx).getPersistenceManager(); + Map conversionsByType = new HashMap(); + try { + resp.setContentType("text/html"); + PrintWriter out = resp.getWriter(); + out.println(""); + out.println(""); + out.println("Device stats conversion"); + out.println(""); + out.println(""); + int total = 0; + do { + Query query = pm.newQuery(DeviceInfo.class); + query.setFilter("debug == null"); + query.setRange(0, MAX_ROWS); + @SuppressWarnings("unchecked") + List uncastDevices = (List) query.execute(); + devices = uncastDevices; + if (devices.isEmpty()) { + out.println("

No devices need conversion

"); + } else { + int size = devices.size(); + total += size; + out.println("

Converting " + size + " devices..."); + out.flush(); + for (DeviceInfo device : devices) { + device.setDebug(true); + pm.currentTransaction().begin(); + pm.makePersistent(device); + pm.currentTransaction().commit(); + String type = device.getType(); + Integer converted = conversionsByType.get(type); + if (converted == null) { + converted = 1; + } else { + converted ++; + } + conversionsByType.put(type, converted); + } + } + query.closeAll(); + flushStats(pm, conversionsByType); + out.println(total + " converted so far.

"); + } while (devices != null && ! devices.isEmpty()); + out.println(""); + } finally { + pm.close(); + } + resp.setStatus(HttpServletResponse.SC_OK); + } + + private void flushStats(PersistenceManager pm, Map conversionsByType) { + for (Entry entry : conversionsByType.entrySet()) { + String type = entry.getKey(); + int size = entry.getValue(); + DeviceStats stats = DeviceStats.convertsDevice(pm, type, size); + pm.currentTransaction().begin(); + pm.makePersistent(stats); + pm.currentTransaction().commit(); + } + conversionsByType.clear(); + } + +} diff --git a/appengine/src/com/google/android/chrometophone/server/StatsServlet.java b/appengine/src/com/google/android/chrometophone/server/StatsServlet.java index ea3fea9..309f2b5 100644 --- a/appengine/src/com/google/android/chrometophone/server/StatsServlet.java +++ b/appengine/src/com/google/android/chrometophone/server/StatsServlet.java @@ -19,8 +19,7 @@ import com.google.android.c2dm.server.C2DMessaging; import java.io.IOException; import java.io.PrintWriter; -import java.util.Map; -import java.util.Map.Entry; +import java.util.List; import javax.jdo.PersistenceManager; import javax.servlet.ServletContext; @@ -39,45 +38,59 @@ public class StatsServlet extends HttpServlet { public void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException{ ServletContext ctx = getServletContext(); + List allStats; PersistenceManager pm = C2DMessaging.getPMF(ctx).getPersistenceManager(); - Map stats = DeviceInfo.getDevicesUsage(pm); - pm.close(); - - resp.setContentType("text/html"); - PrintWriter out = resp.getWriter(); - out.println(""); - out.println(""); - out.println("Device stats"); - out.println(""); - out.println(""); - out.println("

Device stats

"); - if (stats.isEmpty()) { - out.println("

No devices registered yet!

"); - } else { - int total = 0; - for (Integer count : stats.values()) { - total += count; - } - - out.println("" + - ""); - for (Entry entry : stats.entrySet()) { - String type = entry.getKey(); - int count = entry.getValue(); - float share = (100*count) / total; + try { + DeviceStats.init(pm); + allStats = DeviceStats.getAll(pm); + resp.setContentType("text/html"); + PrintWriter out = resp.getWriter(); + out.println(""); + out.println(""); + out.println("Device stats"); + out.println(""); + out.println(""); + out.println("

Device stats

"); + if (allStats.isEmpty()) { + out.println("

No devices registered yet!

"); + } else { + int total = 0; + int added = 0; + int deleted = 0; + int converted = 0; + for (DeviceStats stats: allStats) { + total += stats.getTotal(); + added += stats.getAdded(); + deleted += stats.getDeleted(); + converted += stats.getConverted(); + } + out.println("
TypeCountShare
" + + ""); + for (DeviceStats stats: allStats) { + int count = stats.getTotal(); + float share = (total == 0) ? 0 : (100 * count) / total; + out.println("" + + "" + + "" + + "" + + "" + + "" + + ""); + } out.println("" + - "" + - "" + - ""); + "" + + "" + + "" + + "" + + "" + + ""); + out.println("
TypeAddedDeletedConvertedTotalShare
" + stats.getType() + "" + stats.getAdded() + "" + stats.getDeleted() + "" + stats.getConverted() + "" + count + "" + share + "%
" + type + "" + count + "" + share + "%
Total" + added + "" + deleted + "" + converted + "" + total + "100.0%
"); } - out.println("" + - "Total" + - "" + total+ "" + - "100.0%"); - out.println(""); + + out.println(""); + } finally { + pm.close(); } - - out.println(""); resp.setStatus(HttpServletResponse.SC_OK); } } diff --git a/appengine/src/com/google/android/chrometophone/server/UnregisterServlet.java b/appengine/src/com/google/android/chrometophone/server/UnregisterServlet.java index 8e2c297..f800ec4 100644 --- a/appengine/src/com/google/android/chrometophone/server/UnregisterServlet.java +++ b/appengine/src/com/google/android/chrometophone/server/UnregisterServlet.java @@ -43,7 +43,13 @@ public class UnregisterServlet extends HttpServlet { return; } - reqInfo.deleteRegistration(reqInfo.deviceRegistrationID); + // TODO: make sure new app passes deviceType + String deviceType = reqInfo.getParameter("deviceType"); + if (deviceType == null) { + deviceType = "ac2dm"; + } + + reqInfo.deleteRegistration(reqInfo.deviceRegistrationID, deviceType); resp.getWriter().println(OK_STATUS); } } diff --git a/appengine/war/WEB-INF/appengine-web.xml b/appengine/war/WEB-INF/appengine-web.xml index 99e3164..eac50d5 100644 --- a/appengine/war/WEB-INF/appengine-web.xml +++ b/appengine/war/WEB-INF/appengine-web.xml @@ -16,7 +16,7 @@ --> chrometophone - 11 + dev true diff --git a/appengine/war/WEB-INF/web.xml b/appengine/war/WEB-INF/web.xml index a7cff56..f297cd3 100644 --- a/appengine/war/WEB-INF/web.xml +++ b/appengine/war/WEB-INF/web.xml @@ -74,6 +74,12 @@ + + StatsConversionServlet + com.google.android.chrometophone.server.StatsConversionServlet + + + RegisterServlet /register @@ -119,6 +125,11 @@ /admin/stats + + StatsConversionServlet + /admin/convertStats + + dataMessagingServlet /tasks/c2dm