Bug 735083 - batch inserts into Fennec history provider. r=rnewman

mobile/android/base/sync/repositories/android/AndroidBrowserHistoryDataAccessor.java

@@ -4,15 +4,21 @@
 
 package org.mozilla.gecko.sync.repositories.android;
 
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+
 import org.json.simple.JSONArray;
 import org.json.simple.JSONObject;
 import org.mozilla.gecko.db.BrowserContract;
 import org.mozilla.gecko.sync.Logger;
+import org.mozilla.gecko.sync.repositories.NullCursorException;
 import org.mozilla.gecko.sync.repositories.domain.HistoryRecord;
 import org.mozilla.gecko.sync.repositories.domain.Record;
 
 import android.content.ContentValues;
 import android.content.Context;
+import android.database.Cursor;
 import android.net.Uri;
 
 public class AndroidBrowserHistoryDataAccessor extends
@@ -93,4 +99,87 @@ public class AndroidBrowserHistoryDataAccessor extends
   public void closeExtender() {
     dataExtender.close();
   }
+
+  public static String[] GUID_AND_ID = new String[] { BrowserContract.History.GUID, BrowserContract.History._ID };
+
+  /**
+   * Insert records.
+   * <p>
+   * This inserts all the records (using <code>ContentProvider.bulkInsert</code>),
+   * then inserts all the visit information (using the data extender's
+   * <code>bulkInsert</code>, which internally uses a single database
+   * transaction), and then optionally updates the <code>androidID</code> of
+   * each record.
+   *
+   * @param records
+   *          The records to insert.
+   * @param fetchFreshAndroidIDs
+   *          <code>true</code> to update the <code>androidID</code> of each
+   *          record; <code>false</code> to invalidate them all.
+   * @throws NullCursorException
+   */
+  public void bulkInsert(ArrayList<HistoryRecord> records, boolean fetchFreshAndroidIDs) throws NullCursorException {
+    if (records.isEmpty()) {
+      Logger.debug(LOG_TAG, "No records to insert, returning.");
+    }
+
+    int size = records.size();
+    ContentValues[] cvs = new ContentValues[size];
+    String[] guids = new String[size];
+    Map<String, Record> guidToRecord = new HashMap<String, Record>();
+    int index = 0;
+    for (Record record : records) {
+      if (record.guid == null) {
+        throw new IllegalArgumentException("Record with null GUID passed in to bulkInsert.");
+      }
+      cvs[index] = getContentValues(record);
+      guids[index] = record.guid;
+      guidToRecord.put(record.guid, record);
+      index += 1;
+    }
+
+    // First update the history records.
+    int inserted = context.getContentResolver().bulkInsert(getUri(), cvs);
+    if (inserted == size) {
+      Logger.debug(LOG_TAG, "Inserted " + inserted + " records, as expected.");
+    } else {
+      Logger.debug(LOG_TAG, "Inserted " +
+                   inserted + " records but expected " +
+                   size     + " records; continuing to update visits.");
+    }
+    // Then update the history visits.
+    dataExtender.bulkInsert(records);
+
+    // And finally patch up the androidIDs.
+    if (!fetchFreshAndroidIDs) {
+      return;
+    }
+
+    // We do this here to save a few loops.
+    String guidIn = RepoUtils.computeSQLInClause(guids.length, BrowserContract.History.GUID);
+    Cursor cursor = queryHelper.safeQuery("", GUID_AND_ID, guidIn, guids, null);
+    int guidIndex = cursor.getColumnIndexOrThrow(BrowserContract.History.GUID);
+    int androidIDIndex = cursor.getColumnIndexOrThrow(BrowserContract.History._ID);
+
+    try {
+      cursor.moveToFirst();
+      while (!cursor.isAfterLast()) {
+        String guid = cursor.getString(guidIndex);
+        int androidID = cursor.getInt(androidIDIndex);
+        cursor.moveToNext();
+
+        Record record = guidToRecord.get(guid);
+        if (record == null) {
+          // Should never happen!
+          Logger.warn(LOG_TAG, "Failed to update androidID for record with guid " + guid + ".");
+          continue;
+        }
+        record.androidID = androidID;
+      }
+    } finally {
+      if (cursor != null) {
+        cursor.close();
+      }
+    }
+  }
 }

mobile/android/base/sync/repositories/android/AndroidBrowserHistoryDataExtender.java

@@ -1,50 +1,20 @@
-/* ***** BEGIN LICENSE BLOCK *****
- * Version: MPL 1.1/GPL 2.0/LGPL 2.1
- *
- * The contents of this file are subject to the Mozilla Public License Version
- * 1.1 (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * http://www.mozilla.org/MPL/
- *
- * Software distributed under the License is distributed on an "AS IS" basis,
- * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
- * for the specific language governing rights and limitations under the
- * License.
- *
- * The Original Code is Android Sync Client.
- *
- * The Initial Developer of the Original Code is
- * the Mozilla Foundation.
- * Portions created by the Initial Developer are Copyright (C) 2011
- * the Initial Developer. All Rights Reserved.
- *
- * Contributor(s):
- *   Jason Voll <jvoll@mozilla.com>
- *   Richard Newman <rnewman@mozilla.com>
- *
- * Alternatively, the contents of this file may be used under the terms of
- * either the GNU General Public License Version 2 or later (the "GPL"), or
- * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
- * in which case the provisions of the GPL or the LGPL are applicable instead
- * of those above. If you wish to allow use of your version of this file only
- * under the terms of either the GPL or the LGPL, and not to allow others to
- * use your version of this file under the terms of the MPL, indicate your
- * decision by deleting the provisions above and replace them with the notice
- * and other provisions required by the GPL or the LGPL. If you do not delete
- * the provisions above, a recipient may use your version of this file under
- * the terms of any one of the MPL, the GPL or the LGPL.
- *
- * ***** END LICENSE BLOCK ***** */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 package org.mozilla.gecko.sync.repositories.android;
 
+import java.util.ArrayList;
+
 import org.json.simple.JSONArray;
 import org.mozilla.gecko.sync.Logger;
 import org.mozilla.gecko.sync.repositories.NullCursorException;
+import org.mozilla.gecko.sync.repositories.domain.HistoryRecord;
 
 import android.content.ContentValues;
 import android.content.Context;
 import android.database.Cursor;
+import android.database.SQLException;
 import android.database.sqlite.SQLiteDatabase;
 
 public class AndroidBrowserHistoryDataExtender extends CachedSQLiteOpenHelper {
@@ -119,6 +89,30 @@ public class AndroidBrowserHistoryDataExtender extends CachedSQLiteOpenHelper {
     }
   }
 
+  public void bulkInsert(ArrayList<HistoryRecord> records) {
+    SQLiteDatabase db = this.getCachedWritableDatabase();
+    try {
+      db.beginTransaction();
+
+      for (HistoryRecord record : records) {
+        ContentValues cv = new ContentValues();
+        cv.put(COL_GUID, record.guid);
+        if (record.visits == null) {
+          cv.put(COL_VISITS, "[]");
+        } else {
+          cv.put(COL_VISITS, record.visits.toJSONString());
+        }
+        db.insert(TBL_HISTORY_EXT, null, cv);
+      }
+
+      db.setTransactionSuccessful();
+    } catch (SQLException e) {
+      Logger.error(LOG_TAG, "Caught exception in bulkInsert new history visits.", e);
+    } finally {
+      db.endTransaction();
+    }
+  }
+
   /**
    * Fetch a row.
    *

mobile/android/base/sync/repositories/android/AndroidBrowserHistoryRepositorySession.java

@@ -4,12 +4,17 @@
 
 package org.mozilla.gecko.sync.repositories.android;
 
+import java.util.ArrayList;
+
 import org.json.simple.JSONArray;
 import org.json.simple.JSONObject;
 import org.mozilla.gecko.db.BrowserContract;
+import org.mozilla.gecko.sync.Logger;
 import org.mozilla.gecko.sync.repositories.InactiveSessionException;
 import org.mozilla.gecko.sync.repositories.InvalidSessionTransitionException;
+import org.mozilla.gecko.sync.repositories.NoGuidForIdException;
 import org.mozilla.gecko.sync.repositories.NullCursorException;
+import org.mozilla.gecko.sync.repositories.ParentNotFoundException;
 import org.mozilla.gecko.sync.repositories.Repository;
 import org.mozilla.gecko.sync.repositories.delegates.RepositorySessionBeginDelegate;
 import org.mozilla.gecko.sync.repositories.delegates.RepositorySessionFinishDelegate;
@@ -21,11 +26,17 @@ import android.database.Cursor;
 import android.util.Log;
 
 public class AndroidBrowserHistoryRepositorySession extends AndroidBrowserRepositorySession {
-  
+  public static final String LOG_TAG = "ABHistoryRepoSess";
+
   public static final String KEY_DATE = "date";
   public static final String KEY_TYPE = "type";
   public static final long DEFAULT_VISIT_TYPE = 1;
 
+  /**
+   * The number of records to queue for insertion before writing to databases.
+   */
+  public static int INSERT_RECORD_THRESHOLD = 50;
+
   public AndroidBrowserHistoryRepositorySession(Repository repository, Context context) {
     super(repository);
     dbHelper = new AndroidBrowserHistoryDataAccessor(context);
@@ -113,7 +124,7 @@ public class AndroidBrowserHistoryRepositorySession extends AndroidBrowserReposi
   protected Record prepareRecord(Record record) {
     return record;
   }
-  
+
   @Override
   public void abort() {
     ((AndroidBrowserHistoryDataAccessor) dbHelper).closeExtender();
@@ -125,4 +136,85 @@ public class AndroidBrowserHistoryRepositorySession extends AndroidBrowserReposi
     ((AndroidBrowserHistoryDataAccessor) dbHelper).closeExtender();
     super.finish(delegate);
   }
-}
+
+  protected Object recordsBufferMonitor = new Object();
+  protected ArrayList<HistoryRecord> recordsBuffer = new ArrayList<HistoryRecord>();
+
+  /**
+   * Queue record for insertion, possibly flushing the queue.
+   * <p>
+   * Must be called on <code>storeWorkQueue</code> thread! But this is only
+   * called from <code>store</code>, which is called on the queue thread.
+   *
+   * @param record
+   *          A <code>Record</code> with a GUID that is not present locally.
+   * @return The <code>Record</code> to be inserted. <b>Warning:</b> the
+   *         <code>androidID</code> is not valid! It will be set after the
+   *         records are flushed to the database.
+   */
+  @Override
+  protected Record insert(Record record) throws NoGuidForIdException, NullCursorException, ParentNotFoundException {
+    HistoryRecord toStore = (HistoryRecord) prepareRecord(record);
+    toStore.androidID = -111; // Hopefully this special value will make it easy to catch future errors.
+    updateBookkeeping(toStore); // Does not use androidID -- just GUID -> String map.
+    enqueueNewRecord(toStore);
+    return toStore;
+  }
+
+  /**
+   * Batch incoming records until some reasonable threshold is hit or storeDone
+   * is received.
+   * <p>
+   * Must be called on <code>storeWorkQueue</code> thread!
+   *
+   * @param record A <code>Record</code> with a GUID that is not present locally.
+   * @throws NullCursorException
+   */
+  protected void enqueueNewRecord(HistoryRecord record) throws NullCursorException {
+    synchronized (recordsBufferMonitor) {
+      if (recordsBuffer.size() >= INSERT_RECORD_THRESHOLD) {
+        flushNewRecords();
+      }
+      Logger.debug(LOG_TAG, "Enqueuing new record with GUID " + record.guid);
+      recordsBuffer.add(record);
+    }
+  }
+
+  /**
+   * Flush queue of incoming records to database.
+   * <p>
+   * Must be called on <code>storeWorkQueue</code> thread!
+   * <p>
+   * Must be locked by recordsBufferMonitor!
+   * @throws NullCursorException
+   */
+  protected void flushNewRecords() throws NullCursorException {
+    if (recordsBuffer.size() < 1) {
+      Logger.debug(LOG_TAG, "No records to flush, returning.");
+      return;
+    }
+
+    final ArrayList<HistoryRecord> outgoing = recordsBuffer;
+    recordsBuffer = new ArrayList<HistoryRecord>();
+    Logger.debug(LOG_TAG, "Flushing " + outgoing.size() + " records to database.");
+    // TODO: move bulkInsert to AndroidBrowserDataAccessor?
+    ((AndroidBrowserHistoryDataAccessor) dbHelper).bulkInsert(outgoing, false); // Don't need to update any androidIDs.
+  }
+
+  @Override
+  public void storeDone() {
+    storeWorkQueue.execute(new Runnable() {
+      @Override
+      public void run() {
+        synchronized (recordsBufferMonitor) {
+          try {
+            flushNewRecords();
+          } catch (NullCursorException e) {
+            Logger.warn(LOG_TAG, "Error flushing records to database.", e);
+          }
+        }
+        storeDone(System.currentTimeMillis());
+      }
+    });
+  }
+}

mobile/android/base/sync/repositories/android/AndroidBrowserRepositoryDataAccessor.java

@@ -19,7 +19,7 @@ public abstract class AndroidBrowserRepositoryDataAccessor {
   private static final String[] GUID_COLUMNS = new String[] { BrowserContract.SyncColumns.GUID };
   protected Context context;
   protected static String LOG_TAG = "BrowserDataAccessor";
-  private final RepoUtils.QueryHelper queryHelper;
+  protected final RepoUtils.QueryHelper queryHelper;
 
   public AndroidBrowserRepositoryDataAccessor(Context context) {
     this.context = context;