Bug 1205835 - Create telemetry upload service and upload in onStart. r=rnewman

--HG--
extra : commitid : 53uxpxHX6ek
extra : rebase_source : 16ec581bf868cbc1feb2493c7e6f62ae1675d77b

mobile/android/base/AndroidManifest.xml.in

@@ -469,6 +469,9 @@
             android:name="org.mozilla.gecko.dlc.DownloadContentService">
         </service>
 
+        <service
+          android:name="org.mozilla.gecko.telemetry.TelemetryUploadService"
+          android:exported="false"/>
 
 #include ../services/manifests/FxAccountAndroidManifest_services.xml.in
 

mobile/android/base/java/org/mozilla/gecko/BrowserApp.java

@@ -62,6 +62,8 @@ import org.mozilla.gecko.tabs.TabHistoryController.OnShowTabHistory;
 import org.mozilla.gecko.tabs.TabHistoryFragment;
 import org.mozilla.gecko.tabs.TabHistoryPage;
 import org.mozilla.gecko.tabs.TabsPanel;
+import org.mozilla.gecko.telemetry.TelemetryConstants;
+import org.mozilla.gecko.telemetry.TelemetryUploadService;
 import org.mozilla.gecko.toolbar.AutocompleteHandler;
 import org.mozilla.gecko.toolbar.BrowserToolbar;
 import org.mozilla.gecko.toolbar.BrowserToolbar.TabEditingState;
@@ -159,6 +161,7 @@ import java.util.EnumSet;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
+import java.util.UUID;
 import java.util.Vector;
 
 public class BrowserApp extends GeckoApp
@@ -994,7 +997,8 @@ public class BrowserApp extends GeckoApp
         ThreadUtils.postToBackgroundThread(new Runnable() {
             @Override
             public void run() {
-                if (getProfile().inGuestMode()) {
+                final GeckoProfile profile = getProfile();
+                if (profile.inGuestMode()) {
                     GuestSession.showNotification(BrowserApp.this);
                 } else {
                     // If we're restarting, we won't destroy the activity.
@@ -1002,6 +1006,15 @@ public class BrowserApp extends GeckoApp
                     // have been shown.
                     GuestSession.hideNotification(BrowserApp.this);
                 }
+
+                // We don't upload in onCreate because that's only called when the Activity needs to be instantiated
+                // and it's possible the system will never free the Activity from memory.
+                //
+                // We don't upload in onResume/onPause because that will be called each time the Activity is obscured,
+                // including by our own Activities/dialogs, and there is no reason to upload each time we're unobscured.
+                //
+                // So we're left with onStart/onStop.
+                uploadTelemetry(profile);
             }
         });
     }
@@ -3919,6 +3932,26 @@ public class BrowserApp extends GeckoApp
         mDynamicToolbar.setTemporarilyVisible(false, VisibilityTransition.IMMEDIATE);
     }
 
+    private void uploadTelemetry(final GeckoProfile profile) {
+        if (!TelemetryConstants.UPLOAD_ENABLED || profile.inGuestMode()) {
+            return;
+        }
+
+        final SharedPreferences sharedPrefs = GeckoSharedPrefs.forProfileName(this, profile.getName());
+        final int seq = sharedPrefs.getInt(TelemetryConstants.PREF_SEQ_COUNT, 1);
+
+        final Intent i = new Intent(TelemetryConstants.ACTION_UPLOAD_CORE);
+        i.setClass(this, TelemetryUploadService.class);
+        i.putExtra(TelemetryConstants.EXTRA_DOC_ID, UUID.randomUUID().toString());
+        i.putExtra(TelemetryConstants.EXTRA_PROFILE_NAME, profile.getName());
+        i.putExtra(TelemetryConstants.EXTRA_PROFILE_PATH, profile.getDir().toString());
+        i.putExtra(TelemetryConstants.EXTRA_SEQ, seq);
+        startService(i);
+
+        // Intent redelivery will ensure this value gets used - see TelemetryUploadService class comments for details.
+        sharedPrefs.edit().putInt(TelemetryConstants.PREF_SEQ_COUNT, seq + 1).apply();
+    }
+
     public static interface Refreshable {
         public void refresh();
     }

mobile/android/base/java/org/mozilla/gecko/telemetry/TelemetryConstants.java

@@ -4,8 +4,26 @@
 
 package org.mozilla.gecko.telemetry;
 
+import org.mozilla.gecko.AppConstants;
+
 public class TelemetryConstants {
 
+    // Change these two values to enable upload in developer builds.
+    public static final boolean UPLOAD_ENABLED = AppConstants.MOZILLA_OFFICIAL; // Disabled for developer builds.
+    public static final String DEFAULT_SERVER_URL = "https://incoming.telemetry.mozilla.org";
+
+    public static final String USER_AGENT =
+            "Firefox-Android-Telemetry/" + AppConstants.MOZ_APP_VERSION + " (" + AppConstants.MOZ_APP_UA_NAME + ")";
+
+    public static final String ACTION_UPLOAD_CORE = "uploadCore";
+    public static final String EXTRA_DOC_ID = "docId";
+    public static final String EXTRA_PROFILE_NAME = "geckoProfileName";
+    public static final String EXTRA_PROFILE_PATH = "geckoProfilePath";
+    public static final String EXTRA_SEQ = "seq";
+
+    public static final String PREF_SERVER_URL = "telemetry-serverUrl";
+    public static final String PREF_SEQ_COUNT = "telemetry-seqCount";
+
     public static class CorePing {
         private CorePing() { /* To prevent instantiation */ }
 

mobile/android/base/java/org/mozilla/gecko/telemetry/TelemetryUploadService.java

@@ -0,0 +1,220 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+package org.mozilla.gecko.telemetry;
+
+import android.content.Intent;
+import android.content.SharedPreferences;
+import android.support.annotation.NonNull;
+import android.util.Log;
+import ch.boye.httpclientandroidlib.HttpResponse;
+import ch.boye.httpclientandroidlib.client.ClientProtocolException;
+import org.mozilla.gecko.GeckoProfile;
+import org.mozilla.gecko.GeckoSharedPrefs;
+import org.mozilla.gecko.background.BackgroundService;
+import org.mozilla.gecko.sync.net.BaseResource;
+import org.mozilla.gecko.sync.net.BaseResourceDelegate;
+import org.mozilla.gecko.sync.net.Resource;
+import org.mozilla.gecko.util.StringUtils;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.security.GeneralSecurityException;
+
+/**
+ * The service that handles uploading telemetry payloads to the server.
+ *
+ * Note that we'll fail to upload if the network is off or background uploads are disabled but the caller is still
+ * expected to increment the sequence number.
+ */
+public class TelemetryUploadService extends BackgroundService {
+    private static final String LOGTAG = StringUtils.safeSubstring("Gecko" + TelemetryUploadService.class.getSimpleName(), 0, 23);
+    private static final String WORKER_THREAD_NAME = LOGTAG + "Worker";
+
+    public TelemetryUploadService() {
+        super(WORKER_THREAD_NAME);
+
+        // Intent redelivery can fail hard (e.g. we OOM as we try to upload, the Intent gets redelivered, repeat) so for
+        // simplicity, we avoid it for now. In the unlikely event that Android kills our upload service, we'll thus fail
+        // to upload the document with a specific sequence number. Furthermore, we never attempt to re-upload it.
+        //
+        // We'll fix this issue in bug 1243585.
+        setIntentRedelivery(false);
+    }
+
+    /**
+     * Handles a core ping with the mandatory extras:
+     *   EXTRA_DOC_ID: a unique document ID.
+     *   EXTRA_SEQ: a sequence number for this upload.
+     *   EXTRA_PROFILE_NAME: the gecko profile name.
+     *   EXTRA_PROFILE_PATH: the gecko profile path.
+     *
+     * Note that for a given doc ID, seq should always be identical because these are the tools the server uses to
+     * de-duplicate documents. In order to maintain this consistency, we receive the doc ID and seq from the Intent and
+     * rely on the caller to update the values. The Service can be killed at any time so we can't ensure seq could be
+     * incremented properly if we tried to do so in the Service.
+     */
+    @Override
+    public void onHandleIntent(final Intent intent) {
+        Log.d(LOGTAG, "Service started");
+
+        if (!TelemetryConstants.UPLOAD_ENABLED) {
+            Log.d(LOGTAG, "Telemetry upload feature is compile-time disabled; not handling upload intent.");
+            return;
+        }
+
+        if (!isReadyToUpload(intent)) {
+            return;
+        }
+
+        if (!TelemetryConstants.ACTION_UPLOAD_CORE.equals(intent.getAction())) {
+            Log.w(LOGTAG, "Unknown action: " + intent.getAction() + ". Returning");
+            return;
+        }
+
+        final String docId = intent.getStringExtra(TelemetryConstants.EXTRA_DOC_ID);
+        final int seq = intent.getIntExtra(TelemetryConstants.EXTRA_SEQ, -1);
+
+        final String profileName = intent.getStringExtra(TelemetryConstants.EXTRA_PROFILE_NAME);
+        final String profilePath = intent.getStringExtra(TelemetryConstants.EXTRA_PROFILE_PATH);
+
+        uploadCorePing(docId, seq, profileName, profilePath);
+    }
+
+    private boolean isReadyToUpload(final Intent intent) {
+        // Intent can be null. Bug 1025937.
+        if (intent == null) {
+            Log.d(LOGTAG, "Received null intent. Returning.");
+            return false;
+        }
+
+        // Don't do anything if the device can't talk to the server.
+        if (!backgroundDataIsEnabled()) {
+            Log.d(LOGTAG, "Background data is not enabled; skipping.");
+            return false;
+        }
+
+        if (intent.getStringExtra(TelemetryConstants.EXTRA_DOC_ID) == null) {
+            Log.w(LOGTAG, "Received invalid doc ID in Intent. Returning");
+            return false;
+        }
+
+        if (!intent.hasExtra(TelemetryConstants.EXTRA_SEQ)) {
+            Log.w(LOGTAG, "Received Intent without sequence number. Returning");
+            return false;
+        }
+
+        if (intent.getStringExtra(TelemetryConstants.EXTRA_PROFILE_NAME) == null) {
+            Log.w(LOGTAG, "Received invalid profile name in Intent. Returning");
+            return false;
+        }
+
+        // GeckoProfile can use the name to get the path so this isn't strictly necessary.
+        // However, getting the path requires parsing an ini file so we optimize by including it here.
+        if (intent.getStringExtra(TelemetryConstants.EXTRA_PROFILE_PATH) == null) {
+            Log.w(LOGTAG, "Received invalid profile path in Intent. Returning");
+            return false;
+        }
+
+        return true;
+    }
+
+    private void uploadCorePing(@NonNull final String docId, final int seq, @NonNull final String profileName,
+                @NonNull final String profilePath) {
+        final GeckoProfile profile = GeckoProfile.get(this, profileName, profilePath);
+
+        final String clientId;
+        try {
+            clientId = profile.getClientId();
+        } catch (final IOException e) {
+            // Don't log the exception to avoid leaking the profile path.
+            Log.w(LOGTAG, "Unable to get client ID to generate core ping: returning.");
+            return;
+        }
+
+        // Each profile can have different telemetry data so we intentionally grab the shared prefs for the profile.
+        final SharedPreferences sharedPrefs = GeckoSharedPrefs.forProfileName(this, profileName);
+        // TODO (bug 1241685): Sync this preference with the gecko preference.
+        final String serverURLSchemeHostPort =
+                sharedPrefs.getString(TelemetryConstants.PREF_SERVER_URL, TelemetryConstants.DEFAULT_SERVER_URL);
+
+        final TelemetryPing corePing =
+                TelemetryPingGenerator.createCorePing(docId, clientId, serverURLSchemeHostPort, seq);
+        final CorePingResultDelegate resultDelegate = new CorePingResultDelegate();
+        uploadPing(corePing, resultDelegate);
+    }
+
+    private void uploadPing(final TelemetryPing ping, final ResultDelegate delegate) {
+        final BaseResource resource;
+        try {
+            resource = new BaseResource(ping.getURL());
+        } catch (final URISyntaxException e) {
+            Log.w(LOGTAG, "URISyntaxException for server URL when creating BaseResource: returning.");
+            return;
+        }
+
+        delegate.setResource(resource);
+        resource.delegate = delegate;
+
+        // We're in a background thread so we don't have any reason to do this asynchronously.
+        // If we tried, onStartCommand would return and IntentService might stop itself before we finish.
+        resource.postBlocking(ping.getPayload());
+    }
+
+    private static class CorePingResultDelegate extends ResultDelegate {
+        public CorePingResultDelegate() {
+            super();
+        }
+
+        @Override
+        public String getUserAgent() {
+            return TelemetryConstants.USER_AGENT;
+        }
+
+        @Override
+        public void handleHttpResponse(final HttpResponse response) {
+            final int status = response.getStatusLine().getStatusCode();
+            switch (status) {
+                case 200:
+                case 201:
+                    Log.d(LOGTAG, "Telemetry upload success.");
+                    break;
+                default:
+                    Log.w(LOGTAG, "Telemetry upload failure. HTTP status: " + status);
+            }
+        }
+
+        @Override
+        public void handleHttpProtocolException(final ClientProtocolException e) {
+            // We don't log the exception to prevent leaking user data.
+            Log.w(LOGTAG, "HttpProtocolException when trying to upload telemetry");
+        }
+
+        @Override
+        public void handleHttpIOException(final IOException e) {
+            // We don't log the exception to prevent leaking user data.
+            Log.w(LOGTAG, "HttpIOException when trying to upload telemetry");
+        }
+
+        @Override
+        public void handleTransportException(final GeneralSecurityException e) {
+            // We don't log the exception to prevent leaking user data.
+            Log.w(LOGTAG, "Transport exception when trying to upload telemetry");
+        }
+    }
+
+    /**
+     * A hack because I want to set the resource after the Delegate is constructed.
+     * Be sure to call {@link #setResource(Resource)}!
+     */
+    private static abstract class ResultDelegate extends BaseResourceDelegate {
+        public ResultDelegate() {
+            super(null);
+        }
+
+        protected void setResource(final Resource resource) {
+            this.resource = resource;
+        }
+    }
+}

mobile/android/base/moz.build

@@ -543,6 +543,7 @@ gbjar.sources += ['java/org/mozilla/gecko/' + x for x in [
     'telemetry/TelemetryConstants.java',
     'telemetry/TelemetryPing.java',
     'telemetry/TelemetryPingGenerator.java',
+    'telemetry/TelemetryUploadService.java',
     'TelemetryContract.java',
     'TextSelection.java',
     'TextSelectionHandle.java',

mobile/android/services/src/main/java/org/mozilla/gecko/sync/net/BaseResource.java

@@ -497,6 +497,16 @@ public class BaseResource implements Resource {
     post(jsonEntity(o));
   }
 
+  /**
+   * Perform an HTTP POST as with {@link BaseResource#post(ExtendedJSONObject)}, returning only
+   * after callbacks have been invoked.
+   */
+  public void postBlocking(final ExtendedJSONObject o) {
+    // Until we use the asynchronous Apache HttpClient, we can simply call
+    // through.
+    post(jsonEntity(o));
+  }
+
   public void post(JSONObject jsonObject) throws UnsupportedEncodingException {
     post(jsonEntity(jsonObject));
   }