mirror of
https://github.com/mozilla/gecko-dev.git
synced 2024-11-28 07:13:20 +00:00
141 lines
4.8 KiB
Python
141 lines
4.8 KiB
Python
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
from __future__ import absolute_import
|
|
import random
|
|
import os
|
|
import asyncio
|
|
from arsenic.errors import UnknownError, UnknownArsenicError
|
|
|
|
|
|
from condprof.util import logger, get_credentials
|
|
from condprof.helpers import TabSwitcher, execute_async_script, is_mobile
|
|
|
|
|
|
BOOKMARK_FREQUENCY = 5
|
|
MAX_URLS = 150
|
|
MAX_BOOKMARKS = 250
|
|
CallErrors = asyncio.TimeoutError, UnknownError, UnknownArsenicError
|
|
|
|
|
|
class Builder:
|
|
def __init__(self, options):
|
|
self.options = options
|
|
self.words = self._read_lines("words.txt")
|
|
self.urls = self._build_url_list(self._read_lines("urls.txt"))
|
|
self.sync_js = self._load_script("sync")
|
|
self.max_bookmarks = options.get("max_bookmarks", MAX_BOOKMARKS)
|
|
self.bookmark_js = self._load_script("bookmark")
|
|
self.platform = options.get("platform", "")
|
|
self.mobile = is_mobile(self.platform)
|
|
self.max_urls = options.get("max_urls", MAX_URLS)
|
|
|
|
# see Bug 1608604 & see Bug 1619107 - we have stability issues @ bitbar
|
|
if self.mobile:
|
|
self.max_urls = min(self.max_urls, 20)
|
|
|
|
logger.info("platform: %s" % self.platform)
|
|
logger.info("max_urls: %s" % self.max_urls)
|
|
self.bookmark_frequency = options.get("bookmark_frequency", BOOKMARK_FREQUENCY)
|
|
|
|
# we're syncing only on desktop for now
|
|
self.syncing = not self.mobile
|
|
if self.syncing:
|
|
self.username, self.password = get_credentials()
|
|
if self.username is None:
|
|
raise ValueError("Sync operations need an FxA username and password")
|
|
else:
|
|
self.username, self.password = None, None
|
|
|
|
def _load_script(self, name):
|
|
return "\n".join(self._read_lines("%s.js" % name))
|
|
|
|
def _read_lines(self, filename):
|
|
path = os.path.join(os.path.dirname(__file__), filename)
|
|
with open(path) as f:
|
|
return f.readlines()
|
|
|
|
def _build_url_list(self, urls):
|
|
url_list = []
|
|
for url in urls:
|
|
url = url.strip()
|
|
if url.startswith("#"):
|
|
continue
|
|
for word in self.words:
|
|
word = word.strip()
|
|
if word.startswith("#"):
|
|
continue
|
|
url_list.append((url.format(word), word))
|
|
random.shuffle(url_list)
|
|
return url_list
|
|
|
|
async def add_bookmark(self, session, url, title):
|
|
logger.info("Adding bookmark to %s" % url)
|
|
return await execute_async_script(
|
|
session, self.bookmark_js, url, title, self.max_bookmarks
|
|
)
|
|
|
|
async def sync(self, session, metadata):
|
|
if not self.syncing:
|
|
return
|
|
# now that we've visited all pages, we want to upload to FXSync
|
|
logger.info("Syncing profile to FxSync")
|
|
logger.info("Username is %s, password is %s" % (self.username, self.password))
|
|
script_res = await execute_async_script(
|
|
session,
|
|
self.sync_js,
|
|
self.username,
|
|
self.password,
|
|
"https://accounts.stage.mozaws.net",
|
|
)
|
|
if script_res is None:
|
|
script_res = {}
|
|
metadata["logs"] = script_res.get("logs", {})
|
|
metadata["result"] = script_res.get("result", 0)
|
|
metadata["result_message"] = script_res.get("result_message", "SUCCESS")
|
|
return metadata
|
|
|
|
async def _visit_url(self, current, session, url, word):
|
|
await asyncio.wait_for(session.get(url), 5)
|
|
if current % self.bookmark_frequency == 0 and not self.mobile:
|
|
await asyncio.wait_for(self.add_bookmark(session, url, word), 5)
|
|
|
|
async def __call__(self, session):
|
|
metadata = {}
|
|
|
|
tabs = TabSwitcher(session, self.options)
|
|
await tabs.create_windows()
|
|
visited = 0
|
|
|
|
for current, (url, word) in enumerate(self.urls):
|
|
logger.info("%d/%d %s" % (current + 1, self.max_urls, url))
|
|
retries = 0
|
|
while retries < 3:
|
|
try:
|
|
await self._visit_url(current, session, url, word)
|
|
visited += 1
|
|
break
|
|
except CallErrors:
|
|
await asyncio.sleep(1.0)
|
|
retries += 1
|
|
|
|
if current == self.max_urls - 1:
|
|
break
|
|
|
|
# switch to the next tab
|
|
try:
|
|
await asyncio.wait_for(tabs.switch(), 5)
|
|
except CallErrors:
|
|
# if we can't switch, it's ok
|
|
pass
|
|
|
|
metadata["visited_url"] = visited
|
|
await self.sync(session, metadata)
|
|
return metadata
|
|
|
|
|
|
async def full(session, options):
|
|
builder = Builder(options)
|
|
return await builder(session)
|