torproject/gettor
1
0
Fork 0
mirror of https://github.com/torproject/gettor.git synced 2025-02-12 22:38:57 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Import ilv PR introduce major refactoring and restructure to use Twisted

Browse Source 
Update code and repository structure
Lay foundation to add testing and testing coverage
This commit is contained in:
hiro 2019-02-05 19:12:59 +01:00
parent ba19003400
commit bff816b0d1
42 changed files with 46192 additions and 1271 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
.coveragerc Normal file
View File

@ -0,0 +1,34 @@
[run]
source = gettor
branch = True
#parallel = True
timid = False
[report]
omit =
*/_langs*
*/_version*
*/__init__*
*/sitecustomize*
*/test/*
# Regexes for lines to exclude from report generation:
exclude_lines =
pragma: no cover
# don't complain if the code doesn't hit unimplemented sections:
raise NotImplementedError
pass
# don't complain if non-runnable or debuging code isn't run:
if 0:
if False:
if self[.verbosity.]
if options[.verbosity.]
def __repr__
if __name__ == .__main__.:
except Exception as impossible:
# Ignore source code which cannot be found:
ignore_errors = True
# Exit with status code 2 if under this percentage is covered:
fail_under = 80
[html]
directory = doc/coverage-html

5
.gitignore vendored
View File

@ -0,0 +1,5 @@
venv
__pycache__
*.pyc
log
gettor.db

1
AUTHORS
View File

@ -1,4 +1,5 @@
Current maintainer/core developers:
hiro <hiro@torproject.org>
Israel Leiva <ilv@torproject.org> 4096R/540BFC0E
Past core developers:

34
Makefile
View File

@ -0,0 +1,34 @@
.PHONY: install test
.DEFAULT: install test
TRIAL:=$(shell which trial)
VERSION:=$(shell git describe)
define PYTHON_WHICH
import platform
import sys
sys.stdout.write(platform.python_implementation())
endef
PYTHON_IMPLEMENTATION:=$(shell python3 -c '$(PYTHON_WHICH)')
test:
python3 setup.py test
coverage-test:
ifeq ($(PYTHON_IMPLEMENTATION),PyPy)
@echo "Detected PyPy... not running coverage."
python setup.py test
else
coverage run --rcfile=".coveragerc" $(TRIAL) ./test/test_*.py
coverage report --rcfile=".coveragerc"
endif
coverage-html:
coverage html --rcfile=".coveragerc"
coverage: coverage-test coverage-html
tags:
find ./gettor -type f -name "*.py" -print | xargs etags

44745
TAGS Normal file
View File

File diff suppressed because it is too large Load Diff

39
bin/gettor_service Executable file
View File

@ -0,0 +1,39 @@
#!/bin/bash
#
# This file is part of GetTor, a Tor Browser distribution system.
#
# :authors: isra <ilv@torproject.org>
# see also AUTHORS file
#
# :copyright: (c) 2008-2014, The Tor Project, Inc.
# (c) 2014-2018, Israel Leiva
#
# :license: This is Free Software. See LICENSE for license information.
source venv/bin/activate
case "$1" in
start)
twistd3 --python=scripts/gettor --logfile=log/gettor.log --pidfile=gettor.pid
;;
stop)
kill -INT `cat gettor.pid`
;;
restart)
$0 stop
sleep 2;
$0 start
;;
status)
if [ -e gettor.pid ]; then
echo gettor is running with pid=`cat gettor.pid`
else
echo gettor is NOT running
exit 1
fi
;;
*)
echo "Usage: $0 {start|stop|status|restart}"
esac
exit 0

6
config/blacklist.cfg
View File

@ -1,6 +0,0 @@
[general]
db: /path/to/gettor.db
[log]
level: DEBUG
dir: /path/to/log

12
config/core.cfg
View File

@ -1,12 +0,0 @@
[general]
basedir: /path/to/gettor
db: gettor.db
[links]
dir: /path/to/providers/
os: linux,windows,osx
locales: es,en
[log]
dir: /path/to/log/
level: DEBUG

17
config/smtp.cfg
View File

@ -1,17 +0,0 @@
[general]
basedir: /path/to/gettor/smtp
mirrors: /path/to/mirrors
our_domain: torproject.org
core_cfg: /path/to/core.cfg
[blacklist]
cfg: /path/to/blacklist.cfg
max_requests: 3
wait_time: 20
[i18n]
dir: /path/to/i18n/
[log]
level: DEBUG
dir: /path/to/log/

0
config/tor-mirrors → config/tor-mirrors.json
View File

21
config/xmpp.cfg
View File

@ -1,21 +0,0 @@
[account]
user: account@domain
password:
[general]
basedir: /path/to/gettor/xmpp/
core_cfg: /path/to/core.cfg
max_words: 10
db: /path/to/gettor.db
[blacklist]
cfg: /path/to/blacklist.cfg
max_requests: 3
wait_time: 20
[i18n]
dir: /path/to/i18n/
[log]
level: DEBUG
dir: /path/to/log/

58
dev_scripts/create_db.py
View File

@ -1,58 +0,0 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# This file is part of GetTor, a Tor Browser distribution system.
#
# :authors: Israel Leiva <ilv@riseup.net>
# see also AUTHORS file
#
# :copyright: (c) 2008-2014, The Tor Project, Inc.
# (c) 2014, Israel Leiva
#
# :license: This is Free Software. See LICENSE for license information.
import os
import sqlite3
import argparse
def main():
"""Create/delete GetTor database for managing stats and blacklisting.
Database file (.db) must be empty. If it doesn't exist, it will be
created. See argparse usage for more details.
"""
parser = argparse.ArgumentParser(description='Utility for GetTor'
' database')
parser.add_argument('-c', '--create', default=None,
metavar='path_to_database.db',
help='create database')
parser.add_argument('-d', '--delete', default=None,
metavar='path_to_database.db',
help='delete database')
args = parser.parse_args()
if args.create:
con = sqlite3.connect(args.create)
with con:
cur = con.cursor()
# table for handling users (i.e. blacklist)
cur.execute(
"CREATE TABLE users(id TEXT, service TEXT, times INT,"
"blocked INT, last_request TEXT)"
)
cur.execute(
"CREATE TABLE requests(date TEXT, request TEXT, os TEXT,"
" locale TEXT, channel TEXT, PRIMARY KEY (date, channel))"
)
print "Database %s created" % os.path.abspath(args.create)
elif args.delete:
os.remove(os.path.abspath(args.delete))
print "Database %s deleted" % os.path.abspath(args.delete)
else:
print "See --help for details on usage."
if __name__ == "__main__":
main()

0
INSTALL-SMTP → doc/INSTALL-SMTP
View File

10
gettor.conf.json Normal file
View File

@ -0,0 +1,10 @@
{
"platforms": ["linux", "osx", "windows"],
"dbname": "gettor.db",
"email_parser_logfile": "email_parser.log",
"email_requests_limit": 5,
"sendmail_interval": 10,
"sendmail_addr": "email@addr",
"sendmail_host": "host",
"sendmail_port": 587
}

18
gettor/__init__.py
View File

@ -1 +1,17 @@
# yes it's empty, of such a fullness
# -*- coding: utf-8 -*-
"""
This file is part of GetTor, a service providing alternative methods to download
the Tor Browser.
:authors: Hiro <hiro@torproject.org>
please also see AUTHORS file
:copyright: (c) 2008-2014, The Tor Project, Inc.
(c) 2014, all entities within the AUTHORS file
:license: see included LICENSE for information
"""
from .utils import strings
__version__ = strings.get_version()
__locales__ = strings.get_locales()

481
gettor/core.py
View File

@ -1,481 +0,0 @@
# -*- coding: utf-8 -*-
#
# This file is part of GetTor, a Tor Browser distribution system.
#
# :authors: Israel Leiva <ilv@riseup.net>
# see also AUTHORS file
#
# :copyright: (c) 2008-2014, The Tor Project, Inc.
# (c) 2014, Israel Leiva
#
# :license: This is Free Software. See LICENSE for license information.
import os
import re
import logging
import gettext
import tempfile
import ConfigParser
import db
import utils
"""Core module for getting links from providers."""
class ConfigError(Exception):
pass
class NotSupportedError(Exception):
pass
class LinkFormatError(Exception):
pass
class LinkFileError(Exception):
pass
class InternalError(Exception):
pass
class Core(object):
"""Get links from providers and deliver them to other modules.
Public methods:
get_links(): Get the links for the OS and locale requested.
create_links_file(): Create a file to store links of a provider.
add_link(): Add a link to a links file of a provider.
get_supported_os(): Get a list of supported operating systems.
get_supported_lc(): Get a list of supported locales.
Exceptions:
UnsupportedOSError: OS and/or locale not supported.
ConfigError: Something's misconfigured.
LinkFormatError: The link added doesn't seem legit.
LinkFileError: Error related to the links file of a provider.
InternalError: Something went wrong internally.
"""
def __init__(self, cfg=None):
"""Create a new core object by reading a configuration file.
:param: cfg (string) the path of the configuration file.
:raise: ConfigurationError if the configuration file doesn't exists
or if something goes wrong while reading options from it.
"""
default_cfg = 'core.cfg'
config = ConfigParser.ConfigParser()
if cfg is None or not os.path.isfile(cfg):
cfg = default_cfg
try:
with open(cfg) as f:
config.readfp(f)
except IOError:
raise ConfigError("File %s not found!" % cfg)
try:
self.supported_lc = config.get('links', 'locales')
self.supported_os = config.get('links', 'os')
basedir = config.get('general', 'basedir')
self.linksdir = config.get('links', 'dir')
self.linksdir = os.path.join(basedir, self.linksdir)
self.i18ndir = config.get('i18n', 'dir')
loglevel = config.get('log', 'level')
logdir = config.get('log', 'dir')
logfile = os.path.join(logdir, 'core.log')
dbname = config.get('general', 'db')
dbname = os.path.join(basedir, dbname)
self.db = db.DB(dbname)
except ConfigParser.Error as e:
raise ConfigError("Configuration error: %s" % str(e))
except db.Exception as e:
raise InternalError("%s" % e)
# logging
log = logging.getLogger(__name__)
logging_format = utils.get_logging_format()
date_format = utils.get_date_format()
formatter = logging.Formatter(logging_format, date_format)
log.info('Redirecting CORE logging to %s' % logfile)
logfileh = logging.FileHandler(logfile, mode='a+')
logfileh.setFormatter(formatter)
logfileh.setLevel(logging.getLevelName(loglevel))
log.addHandler(logfileh)
# stop logging on stdout from now on
log.propagate = False
self.log = log
def _get_msg(self, msgid, lc):
"""Get message identified by msgid in a specific locale.
:param: msgid (string) the identifier of a string.
:param: lc (string) the locale.
:return: (string) the message from the .po file.
"""
# obtain the content in the proper language
try:
t = gettext.translation(lc, self.i18ndir, languages=[lc])
_ = t.ugettext
msgstr = _(msgid)
return msgstr
except IOError as e:
raise ConfigError("%s" % str(e))
def get_links(self, service, os, lc):
"""Get links for OS in locale.
This method should be called from the services modules of
GetTor (e.g. SMTP). To make it easy we let the module calling us
specify the name of the service (for stats purpose).
:param: service (string) the service trying to get the links.
:param: os (string) the operating system.
:param: lc (string) tthe locale.
:raise: InternalError if something goes wrong while internally.
:return: (string) the links.
"""
# english and windows by default
if lc not in self.supported_lc:
self.log.debug("Request for locale not supported. Default to en")
lc = 'en'
if os not in self.supported_os:
self.log.debug("Request for OS not supported. Default to windows")
os = 'windows'
# this could change in the future, let's leave it isolated.
self.log.debug("Trying to get the links...")
try:
links = self._get_links(os, lc)
self.log.debug("OK")
except InternalError as e:
self.log.debug("FAILED")
raise InternalError("%s" % str(e))
if links is None:
self.log.debug("No links found")
raise InternalError("No links. Something is wrong.")
return links
def _get_links(self, osys, lc):
"""Internal method to get the links.
Looks for the links inside each provider file. This should only be
called from get_links() method.
:param: osys (string) the operating system.
:param: lc (string) the locale.
:return: (string/None) links on success, None otherwise.
"""
# read the links files using ConfigParser
# see the README for more details on the format used
links_files = []
links32 = {}
links64 = {}
# for the message to be sent
if osys == 'windows':
arch = '32/64'
elif osys == 'osx':
arch = '64'
else:
arch = '32'
# look for files ending with .links
p = re.compile('.*\.links$')
for name in os.listdir(self.linksdir):
path = os.path.abspath(os.path.join(self.linksdir, name))
if os.path.isfile(path) and p.match(path):
links_files.append(path)
# let's create a dictionary linking each provider with the links
# found for os and lc. This way makes it easy to check if no
# links were found
providers = {}
# separator
spt = '=' * 72
# reading links from providers directory
for name in links_files:
# we're reading files listed on linksdir, so they must exist!
config = ConfigParser.ConfigParser()
# but just in case they don't
try:
with open(name) as f:
config.readfp(f)
except IOError:
raise InternalError("File %s not found!" % name)
try:
pname = config.get('provider', 'name')
# check if current provider pname has links for os in lc
providers[pname] = config.get(osys, lc)
except ConfigParser.Error as e:
# we should at least have the english locale available
self.log.error("Request for %s, returning 'en' instead" % lc)
providers[pname] = config.get(osys, 'en')
try:
#test = providers[pname].split("$")
#self.log.debug(test)
if osys == 'linux':
t32, t64 = [t for t in providers[pname].split(",") if t]
link, signature, chs32 = [l for l in t32.split("$") if l]
link = " %s: %s" % (pname, link)
links32[link] = signature
link, signature, chs64 = [l for l in t64.split("$") if l]
link = " %s: %s" % (pname, link.lstrip())
links64[link] = signature
else:
link, signature, chs32 = [l for l in providers[pname].split("$") if l]
link = " %s: %s" % (pname, link)
links32[link] = signature
#providers[pname] = providers[pname].replace(",", "")
#providers[pname] = providers[pname].replace("$", "\n\n")
### We will improve and add the verification section soon ###
# all packages are signed with same key
# (Tor Browser developers)
# fingerprint = config.get('key', 'fingerprint')
# for now, english messages only
# fingerprint_msg = self._get_msg('fingerprint', 'en')
# fingerprint_msg = fingerprint_msg % fingerprint
except ConfigParser.Error as e:
raise InternalError("%s" % str(e))
# create the final links list with all providers
all_links = []
msg = "Tor Browser %s-bit:\n" % arch
for link in links32:
msg = "%s\n%s" % (msg, link)
all_links.append(msg)
if osys == 'linux':
msg = "\n\n\nTor Browser 64-bit:\n"
for link in links64:
msg = "%s\n%s" % (msg, link)
all_links.append(msg)
### We will improve and add the verification section soon ###
"""
msg = "\n\n\nTor Browser's signature %s-bit:" %\
arch
for link in links32:
msg = "%s\n%s" % (msg, links32[link])
all_links.append(msg)
if osys == 'linux':
msg = "\n\n\nTor Browser's signature 64-bit:"
for link in links64:
msg = "%s%s" % (msg, links64[link])
all_links.append(msg)
msg = "\n\n\nSHA256 of Tor Browser %s-bit (advanced): %s\n" %\
(arch, chs32)
all_links.append(msg)
if osys == 'linux':
msg = "SHA256 of Tor Browser 64-bit (advanced): %s\n" % chs64
all_links.append(msg)
"""
### end verification ###
"""
for key in providers.keys():
# get more friendly description of the provider
try:
# for now, english messages only
provider_desc = self._get_msg('provider_desc', 'en')
provider_desc = provider_desc % key
all_links.append(
"%s\n%s\n\n%s%s\n\n\n" %
(provider_desc, spt, ''.join(providers[key]), spt)
)
except ConfigError as e:
raise InternalError("%s" % str(e))
"""
### We will improve and add the verification section soon ###
# add fingerprint after the links
# all_links.append(fingerprint_msg)
if all_links:
return "".join(all_links)
else:
# we're trying to get supported os an lc
# but there aren't any links!
return None
def get_supported_os(self):
"""Public method to get the list of supported operating systems.
:return: (list) the supported operating systems.
"""
return self.supported_os.split(',')
def get_supported_lc(self):
"""Public method to get the list of supported locales.
:return: (list) the supported locales.
"""
return self.supported_lc.split(',')
def create_links_file(self, provider, fingerprint):
"""Public method to create a links file for a provider.
This should be used by all providers since it writes the links
file with the proper format. It backs up the old links file
(if exists) and creates a new one.
:param: provider (string) the provider (links file will use this
name in slower case).
:param: fingerprint (string) the fingerprint of the key that signed
the packages to be uploaded to the provider.
"""
linksfile = os.path.join(self.linksdir, provider.lower() + '.links')
linksfile_backup = ""
self.log.debug("Request to create a new links file")
if os.path.isfile(linksfile):
self.log.debug("Trying to backup the old one...")
try:
# backup the old file in case something fails
linksfile_backup = linksfile + '.backup'
os.rename(linksfile, linksfile_backup)
except OSError as e:
self.log.debug("FAILED %s" % str(e))
raise LinkFileError(
"Error while creating new links file: %s" % str(e)
)
self.log.debug("Creating empty links file...")
try:
# this creates an empty links file
content = ConfigParser.RawConfigParser()
content.add_section('provider')
content.set('provider', 'name', provider)
content.add_section('key')
content.set('key', 'fingerprint', fingerprint)
content.add_section('linux')
content.add_section('windows')
content.add_section('osx')
with open(linksfile, 'w+') as f:
content.write(f)
except Exception as e:
self.log.debug("FAILED: %s" % str(e))
# if we passed the last exception, then this shouldn't
# be a problem...
if linksfile_backup:
os.rename(linksfile_backup, linksfile)
raise LinkFileError(
"Error while creating new links file: %s" % str(e)
)
def add_link(self, provider, osys, lc, link):
"""Public method to add a link to a provider's links file.
Use ConfigParser to add a link into the os section, under the lc
option. It checks for valid format; the provider's script should
use the right format (see design).
:param: provider (string) the provider.
:param: os (string) the operating system.
:param: lc (string) the locale.
:param: link (string) link to be added.
:raise: NotsupportedError if the OS and/or locale is not supported.
:raise: LinkFileError if there is no links file for the provider.
:raise: LinkFormatError if the link format doesn't seem legit.
:raise: InternalError if the links file doesn't have a section for
the OS requested. This *shouldn't* happen because it means
the file wasn't created correctly.
"""
linksfile = os.path.join(self.linksdir, provider.lower() + '.links')
self.log.debug("Request to add a new link")
# don't try to add unsupported stuff
if lc not in self.supported_lc:
self.log.debug("Request for locale %s not supported" % lc)
raise NotSupportedError("Locale %s not supported" % lc)
if osys not in self.supported_os:
self.log.debug("Request for OS %s not supported" % osys)
raise NotSupportedError("OS %s not supported" % osys)
self.log.debug("Opening links file...")
if os.path.isfile(linksfile):
content = ConfigParser.RawConfigParser()
try:
with open(linksfile) as f:
content.readfp(f)
except IOError as e:
self.log.debug("FAILED %s" % str(e))
raise LinksFileError("File %s not found!" % linksfile)
# check if exists and entry for locale; if not, create it
self.log.debug("Trying to add the link...")
try:
links = content.get(osys, lc)
links = "%s,\n%s" % (links, link)
content.set(osys, lc, links)
self.log.debug("Link added")
with open(linksfile, 'w') as f:
content.write(f)
except ConfigParser.NoOptionError:
content.set(osys, lc, link)
self.log.debug("Link added (with new locale created)")
with open(linksfile, 'w') as f:
content.write(f)
except ConfigParser.NoSectionError as e:
# this shouldn't happen, but just in case
self.log.debug("FAILED (OS not found)")
raise InternalError("Unknown section %s" % str(e))
else:
self.log.debug("FAILED (links file doesn't seem legit)")
raise LinkFileError("No links file for %s" % provider)

38
gettor/main.py Normal file
View File

@ -0,0 +1,38 @@
# -*- coding: utf-8 -*-
"""
This file is part of GetTor, a service providing alternative methods to download
the Tor Browser.
:authors: Hiro <hiro@torproject.org>
please also see AUTHORS file
:copyright: (c) 2008-2014, The Tor Project, Inc.
(c) 2014, all entities within the AUTHORS file
:license: see included LICENSE for information
"""
"""This module sets up GetTor and starts the servers running."""
import sys
from .utils.commons import log
from .utils import options
from .services import BaseService
from .services.email.sendmail import Sendmail
def run(gettor, app):
"""
This is GetTor's main entry point and main runtime loop.
"""
settings = options.parse_settings()
sendmail = Sendmail(settings)
log.info("Starting services.")
sendmail_service = BaseService(
"sendmail", sendmail.get_interval(), sendmail
)
gettor.addService(sendmail_service)
gettor.setServiceParent(app)

1
gettor/parse/__init__.py Normal file
View File

@ -0,0 +1 @@
# empty

217
gettor/parse/email.py Normal file
View File

@ -0,0 +1,217 @@
# -*- coding: utf-8 -*-
#
# This file is part of GetTor, a Tor Browser distribution system.
#
# :authors: isra <ilv@torproject.org>
# see also AUTHORS file
#
# :copyright: (c) 2008-2014, The Tor Project, Inc.
# (c) 2014-2018, Israel Leiva
#
# :license: This is Free Software. See LICENSE for license information.
from __future__ import absolute_import
import re
import dkim
import hashlib
import validate_email
from datetime import datetime
import configparser
from email import message_from_string
from email.utils import parseaddr
from twisted.python import log
from twisted.internet import defer
from twisted.enterprise import adbapi
from .. import PLATFORMS, EMAIL_REQUESTS_LIMIT
from ..db import SQLite3
class AddressError(Exception):
"""
Error if email address is not valid or it can't be normalized.
"""
pass
class DKIMError(Exception):
"""
Error if DKIM signature verification fails.
"""
pass
class EmailParser(object):
"""Class for parsing email requests."""
def __init__(self, to_addr=None, dkim=False):
"""
Constructor.
param (Boolean) dkim: Set dkim verification to True or False.
"""
self.dkim = dkim
self.to_addr = to_addr
def parse(self, msg_str):
"""
Parse message content. Check if email address is well formed, if DKIM
signature is valid, and prevent service flooding. Finally, look for
commands to process the request. Current commands are:
- links: request links for download.
- help: help request.
:param msg_str (str): incomming message as string.
:return dict with email address and command (`links` or `help`).
"""
log.msg("Building email message from string.", system="email parser")
msg = message_from_string(msg_str)
# Normalization will convert <Alice Wonderland> alice@wonderland.net
# into alice@wonderland.net
name, norm_addr = parseaddr(msg['From'])
to_name, norm_to_addr = parseaddr(msg['To'])
log.msg(
"Normalizing and validating FROM email address.",
system="email parser"
)
# Validate_email will do a bunch of regexp to see if the email address
# is well address. Additional options for validate_email are check_mx
# and verify, which check if the SMTP host and email address exist.
# See validate_email package for more info.
if norm_addr and validate_email.validate_email(norm_addr):
log.msg(
"Email address normalized and validated.",
system="email parser"
)
else:
log.err(
"Error normalizing/validating email address.",
system="email parser"
)
raise AddressError("Invalid email address {}".format(msg['From']))
hid = hashlib.sha256(norm_addr)
log.msg(
"Request from {}".format(hid.hexdigest()), system="email parser"
)
if self.to_addr:
if self.to_addr != norm_to_addr:
log.msg("Got request for a different instance of gettor")
log.msg("Intended recipient: {}".format(norm_to_addr))
return {}
# DKIM verification. Simply check that the server has verified the
# message's signature
if self.dkim:
log.msg("Checking DKIM signature.", system="email parser")
# Note: msg.as_string() changes the message to conver it to
# string, so DKIM will fail. Use the original string instead
if dkim.verify(msg_str):
log.msg("Valid DKIM signature.", system="email parser")
else:
log.msg("Invalid DKIM signature.", system="email parser")
username, domain = norm_addr.split("@")
raise DkimError(
"DKIM failed for {} at {}".format(
hid.hexdigest(), domain
)
)
# Search for commands keywords
subject_re = re.compile(r"Subject: (.*)\r\n")
subject = subject_re.search(msg_str)
request = {
"id": norm_addr,
"command": None, "platform": None,
"service": "email"
}
if subject:
subject = subject.group(1)
for word in re.split(r"\s+", subject.strip()):
if word.lower() in PLATFORMS:
request["command"] = "links"
request["platform"] = word.lower()
break
if word.lower() == "help":
request["command"] = "help"
break
if not request["command"]:
for word in re.split(r"\s+", body_str.strip()):
if word.lower() in PLATFORMS:
request["command"] = "links"
request["platform"] = word.lower()
break
if word.lower() == "help":
request["command"] = "help"
break
return request
@defer.inlineCallbacks
def parse_callback(self, request):
"""
Callback invoked when the message has been parsed. It stores the
obtained information in the database for further processing by the
Sendmail service.
:param (dict) request: the built request based on message's content.
It contains the `email_addr` and command `fields`.
:return: deferred whose callback/errback will log database query
execution details.
"""
log.msg(
"Found request for {}.".format(request['command']),
system="email parser"
)
if request["command"]:
now_str = datetime.now().strftime("%Y%m%d%H%M%S")
conn = SQLite3()
hid = hashlib.sha256(request['id'])
# check limits first
num_requests = yield conn.get_num_requests(
id=hid.hexdigest(), service=request['service']
)
if num_requests[0][0] > EMAIL_REQUESTS_LIMIT:
log.msg(
"Discarded. Too many requests from {}.".format(
hid.hexdigest
), system="email parser"
)
else:
conn.new_request(
id=request['id'],
command=request['command'],
platform=request['platform'],
service=request['service'],
date=now_str,
status="ONHOLD",
)
def parse_errback(self, error):
"""
Errback if we don't/can't parse the message's content.
"""
log.msg(
"Error while parsing email content: {}.".format(error),
system="email parser"
)

63
gettor/services/__init__.py Normal file
View File

@ -0,0 +1,63 @@
# -*- coding: utf-8 -*-
"""
This file is part of GetTor, a service providing alternative methods to download
the Tor Browser.
:authors: Hiro <hiro@torproject.org>
please also see AUTHORS file
:copyright: (c) 2008-2014, The Tor Project, Inc.
(c) 2014, all entities within the AUTHORS file
:license: see included LICENSE for information
"""
from __future__ import absolute_import
from twisted.application import internet
from ..utils.commons import log
class BaseService(internet.TimerService):
"""
Base service for Accounts, Messages and Fetchmail. It extends the
TimerService providing asynchronous connection to database by default.
"""
def __init__(self, name, step, instance, *args, **kwargs):
"""
Constructor. Initiate connection to database and link one of Accounts,
Messages or Fetchmail instances to TimerService behavour.
:param name (str): name of the service being initiated (just for log
purposes).
:param step (float): time interval for TimerService, in seconds.
:param instance (object): instance of Accounts, Messages, or
Fetchmail classes.
"""
log.info("SERVICE:: Initializing {} service.".format(name))
self.name = name
self.instance = instance
log.debug("SERVICE:: Initializing TimerService.")
internet.TimerService.__init__(
self, step, self.instance.get_new, **kwargs
)
def startService(self):
"""
Starts the service. Overridden from parent class to add extra logging
information.
"""
log.info("SERVICE:: Starting {} service.".format(self.name))
internet.TimerService.startService(self)
log.info("SERVICE:: Service started.")
def stopService(self):
"""
Stop the service. Overridden from parent class to close connection to
database, shutdown the service and add extra logging information.
"""
log.info("SERVICE:: Stopping {} service.".format(self.name))
log.debug("SERVICE:: Calling shutdown on {}".format(self.name))
self.instance.shutdown()
log.debug("SERVICE:: Shutdown for {} done".format(self.name))
internet.TimerService.stopService(self)
log.info("SERVICE:: Service stopped.")

226
gettor/services/email/sendmail.py Normal file
View File

@ -0,0 +1,226 @@
# -*- coding: utf-8 -*-
#
# This file is part of GetTor, a Tor Browser distribution system.
#
# :authors: isra <ilv@torproject.org>
# see also AUTHORS file
#
# :copyright: (c) 2008-2014, The Tor Project, Inc.
# (c) 2014-2018, Israel Leiva
#
# :license: This is Free Software. See LICENSE for license information.
from __future__ import absolute_import
import gettext
import hashlib
import configparser
from email import encoders
from email import mime
from twisted.internet import defer
from twisted.mail.smtp import sendmail
from ...utils.db import DB
from ...utils.commons import log
class SMTPError(Exception):
"""
Error if we can't send emails.
"""
pass
class Sendmail(object):
"""
Class for sending email replies to `help` and `links` requests.
"""
def __init__(self, settings):
"""
Constructor. It opens and stores a connection to the database.
:dbname: reads from configs
"""
self.settings = settings
dbname = self.settings.get("dbname")
self.conn = DB(dbname)
def get_interval(self):
"""
Get time interval for service periodicity.
:return: time interval (float) in seconds.
"""
return self.settings.get("sendmail_interval")
def sendmail_callback(self, message):
"""
Callback invoked after an email has been sent.
:param message (string): Success details from the server.
"""
log.info("Email sent successfully.")
def sendmail_errback(self, error):
"""
Errback if we don't/can't send the message.
"""
log.debug("Could not send email.")
raise SMTPError("{}".format(error))
def sendmail(self, email_addr, subject, body):
"""
Send an email message. It creates a plain text message, set headers
and content and finally send it.
:param email_addr (str): email address of the recipient.
:param subject (str): subject of the message.
:param content (str): content of the message.
:return: deferred whose callback/errback will handle the SMTP
execution details.
"""
log.debug("Creating plain text email")
message = MIMEText(body)
message['Subject'] = subject
message['From'] = SENDMAIL_ADDR
message['To'] = email_addr
log.debug("Calling asynchronous sendmail.")
return sendmail(
SENDMAIL_HOST, SENDMAIL_ADDR, email_addr, message,
port=SENDMAIL_PORT,
requireAuthentication=True, requireTransportSecurity=True
).addCallback(self.sendmail_callback).addErrback(self.sendmail_errback)
@defer.inlineCallbacks
def get_new(self):
"""
Get new requests to process. This will define the `main loop` of
the Sendmail service.
"""
# Manage help and links messages separately
help_requests = yield self.conn.get_requests(
status="ONHOLD", command="help", service="email"
)
link_requests = yield self.conn.get_requests(
status="ONHOLD", command="links", service="email"
)
if help_requests:
try:
log.info("Got new help request.")
# for now just english
en = gettext.translation(
'email', localedir='locales', languages=['en']
)
en.install()
_ = en.gettext
for request in help_requests:
id = request[0]
date = request[4]
hid = hashlib.sha256(id)
log.info(
"Sending help message to {}.".format(
hid.hexdigest()
)
)
yield self.sendmail(
email_addr=id,
subject=_("help_subject"),
body=_("help_body")
)
yield self.conn.update_stats(
command="help", service="email"
)
yield self.conn.update_request(
id=id, hid=hid.hexdigest(), status="SENT",
service="email", date=date
)
except SMTPError as e:
log.info("Error sending email: {}.".format(e))
elif link_requests:
try:
log.info("Got new links request.")
# for now just english
en = gettext.translation(
'email', localedir='locales', languages=['en']
)
en.install()
_ = en.gettext
for request in link_requests:
id = request[0]
date = request[4]
platform = request[2]
log.info("Getting links for {}.".format(platform))
links = yield self.conn.get_links(
platform=platform, status="ACTIVE"
)
# build message
link_msg = None
for link in links:
provider = link[4]
version = link[3]
arch = link[2]
url = link[0]
link_str = "Tor Browser {} for {}-{} ({}): {}".format(
version, platform, arch, provider, url
)
if link_msg:
link_msg = "{}\n{}".format(link_msg, link_str)
else:
link_msg = link_str
body_msg = _("links_body")
body_msg = body_msg.format(links=link_msg)
subject_msg = _("links_subject")
hid = hashlib.sha256(id)
log.info(
"Sending links to {}.".format(
hid.hexdigest()
)
)
yield self.sendmail(
email_addr=id,
subject=subject_msg,
body=body_msg
)
yield self.conn.update_stats(
command="links", platform=platform, service="email"
)
yield self.conn.update_request(
id=id, hid=hid.hexdigest(), status="SENT",
service="email", date=date
)
except SMTPError as e:
log.info("Error sending email: {}.".format(e))
else:
log.debug("No pending email requests. Keep waiting.")

2
gettor/twitter.py → gettor/services/twitter/twitter.py
View File

@ -16,7 +16,7 @@ import re
import tweepy
import logging
import gettext
import ConfigParser
import configparser
import core
import utils

2
gettor/xmpp.py → gettor/services/xmpp/xmpp.py
View File

@ -17,7 +17,7 @@ import time
import gettext
import hashlib
import logging
import ConfigParser
import configparser
from sleekxmpp import ClientXMPP
from sleekxmpp.xmlstream.stanzabase import JID

535
gettor/smtp.py
View File

@ -1,535 +0,0 @@
# -*- coding: utf-8 -*-
#
# This file is part of GetTor, a Tor Browser distribution system.
#
# :authors: Israel Leiva <ilv@riseup.net>
# see also AUTHORS file
#
# :copyright: (c) 2008-2014, The Tor Project, Inc.
# (c) 2014, Israel Leiva
#
# :license: This is Free Software. See LICENSE for license information.
import os
import re
import sys
import time
import email
import gettext
import logging
import smtplib
import datetime
import ConfigParser
from email import Encoders
from email.MIMEBase import MIMEBase
from email.mime.text import MIMEText
from email.MIMEMultipart import MIMEMultipart
import core
import utils
import blacklist
"""SMTP module for processing email requests."""
OS = {
'osx': 'Mac OS X',
'linux': 'Linux',
'windows': 'Windows'
}
class ConfigError(Exception):
pass
class AddressError(Exception):
pass
class SendEmailError(Exception):
pass
class InternalError(Exception):
pass
class SMTP(object):
"""Receive and reply requests by email.
Public methods:
process_email(): Process the email received.
Exceptions:
ConfigError: Bad configuration.
AddressError: Address of the sender malformed.
SendEmailError: SMTP server not responding.
InternalError: Something went wrong internally.
"""
def __init__(self, cfg=None):
"""Create new object by reading a configuration file.
:param: cfg (string) path of the configuration file.
"""
default_cfg = 'smtp.cfg'
config = ConfigParser.ConfigParser()
if cfg is None or not os.path.isfile(cfg):
cfg = default_cfg
try:
with open(cfg) as f:
config.readfp(f)
except IOError:
raise ConfigError("File %s not found!" % cfg)
try:
self.our_domain = config.get('general', 'our_domain')
self.mirrors = config.get('general', 'mirrors')
self.i18ndir = config.get('i18n', 'dir')
logdir = config.get('log', 'dir')
logfile = os.path.join(logdir, 'smtp.log')
loglevel = config.get('log', 'level')
blacklist_cfg = config.get('blacklist', 'cfg')
self.bl = blacklist.Blacklist(blacklist_cfg)
self.bl_max_req = config.get('blacklist', 'max_requests')
self.bl_max_req = int(self.bl_max_req)
self.bl_wait_time = config.get('blacklist', 'wait_time')
self.bl_wait_time = int(self.bl_wait_time)
core_cfg = config.get('general', 'core_cfg')
self.core = core.Core(core_cfg)
except ConfigParser.Error as e:
raise ConfigError("Configuration error: %s" % str(e))
except blacklist.ConfigError as e:
raise InternalError("Blacklist error: %s" % str(e))
except core.ConfigError as e:
raise InternalError("Core error: %s" % str(e))
# logging
log = logging.getLogger(__name__)
logging_format = utils.get_logging_format()
date_format = utils.get_date_format()
formatter = logging.Formatter(logging_format, date_format)
log.info('Redirecting SMTP logging to %s' % logfile)
logfileh = logging.FileHandler(logfile, mode='a+')
logfileh.setFormatter(formatter)
logfileh.setLevel(logging.getLevelName(loglevel))
log.addHandler(logfileh)
# stop logging on stdout from now on
log.propagate = False
self.log = log
def _is_blacklisted(self, addr):
"""Check if a user is blacklisted.
:param: addr (string) the hashed address of the user.
:return: true is the address is blacklisted, false otherwise.
"""
try:
self.bl.is_blacklisted(
addr, 'SMTP', self.bl_max_req, self.bl_wait_time
)
return False
except blacklist.BlacklistError as e:
return True
def _get_lc(self, addr):
"""Get the locale from an email address.
Process the email received and look for the locale in the recipient
address (e.g. gettor+en@torproject.org). If no locale found, english
by default.
:param: (string) the email address we want to get the locale from.
:return: (string) the locale (english if none).
"""
# if no match found, english by default
lc = 'en'
# look for gettor+locale@torproject.org
m = re.match('gettor\+(\w\w)@\w+\.\w+', addr)
if m:
# we found a request for locale lc
lc = "%s" % m.groups()
return lc.lower()
def _get_normalized_address(self, addr):
"""Get normalized address.
We look for anything inside the last '<' and '>'. Code taken from
the old GetTor (utils.py).
:param: addr (string) the address we want to normalize.
:raise: AddressError if the address can't be normalized.
:return: (string) the normalized address.
"""
if '<' in addr:
idx = addr.rindex('<')
addr = addr[idx:]
m = re.search(r'<([^>]*)>', addr)
if m is None:
# malformed address
raise AddressError("Couldn't extract normalized address "
"from %s" % self_get_sha256(addr))
addr = m.group(1)
return addr
def _get_content(self, email):
"""Get the body content of an email.
:param: email (object) the email object to extract the content from.
:return: (string) body of the message.
"""
# get the body content of the email
maintype = email.get_content_maintype()
if maintype == 'multipart':
for part in email.get_payload():
if part.get_content_maintype() == 'text':
return part.get_payload()
elif maintype == 'text':
return email.get_payload()
def _get_msg(self, msgid, lc):
"""Get message identified by msgid in a specific locale.
:param: msgid (string) the identifier of a string.
:param: lc (string) the locale.
:return: (string) the message from the .po file.
"""
# obtain the content in the proper language
try:
t = gettext.translation(lc, self.i18ndir, languages=[lc])
_ = t.ugettext
msgstr = _(msgid)
return msgstr
except IOError as e:
raise ConfigError("%s" % str(e))
def _parse_email(self, msg, addr):
"""Parse the email received.
Get the locale and parse the text for the rest of the info.
:param: msg (string) the content of the email to be parsed.
:param: addr (string) the address of the recipient (i.e. us).
:return: (list) 4-tuple with locale, os and type of request.
"""
req = self._parse_text(msg)
lc = self._get_lc(addr)
supported_lc = self.core.get_supported_lc()
if lc in supported_lc:
req['lc'] = lc
else:
req['lc'] = 'en'
return req
def _parse_text(self, msg):
"""Parse the text part of the email received.
Try to figure out what the user is asking, namely, the type
of request, the package and os required (if applies).
:param: msg (string) the content of the email to be parsed.
:return: (list) 3-tuple with the type of request, os and pt info.
"""
# by default we asume the request is asking for help
req = {}
req['type'] = 'help'
req['os'] = None
# core knows what OS are supported
supported_os = self.core.get_supported_os()
# search for OS or mirrors request
# if nothing is found, help by default
found_request = False
words = re.split('\s+', msg.strip())
for word in words:
if not found_request:
# OS first
for os in supported_os:
if re.match(os, word, re.IGNORECASE):
req['os'] = os
req['type'] = 'links'
found_request = True
break
# mirrors
if re.match("mirrors?", word, re.IGNORECASE):
req['type'] = 'mirrors'
found_request = True
else:
break
return req
def _create_email(self, from_addr, to_addr, subject, msg):
"""Create an email object.
This object will be used to construct the reply.
:param: from_addr (string) the address of the sender.
:param: to_addr (string) the address of the recipient.
:param: subject (string) the subject of the email.
:param: msg (string) the content of the email.
:return: (object) the email object.
"""
email_obj = MIMEMultipart()
email_obj.set_charset("utf-8")
email_obj['Subject'] = subject
email_obj['From'] = from_addr
email_obj['To'] = to_addr
msg_attach = MIMEText(msg, 'plain')
email_obj.attach(msg_attach)
return email_obj
def _send_email(self, from_addr, to_addr, subject, msg, attach=None):
"""Send an email.
Take a 'from' and 'to' addresses, a subject and the content, creates
the email and send it.
:param: from_addr (string) the address of the sender.
:param: to_addr (string) the address of the recipient.
:param: subject (string) the subject of the email.
:param: msg (string) the content of the email.
:param: attach (string) the path of the mirrors list.
"""
email_obj = self._create_email(from_addr, to_addr, subject, msg)
if(attach):
# for now, the only email with attachment is the one for mirrors
try:
part = MIMEBase('application', "octet-stream")
part.set_payload(open(attach, "rb").read())
Encoders.encode_base64(part)
part.add_header(
'Content-Disposition',
'attachment; filename="mirrors.txt"'
)
email_obj.attach(part)
except IOError as e:
raise SendEmailError('Error with mirrors: %s' % str(e))
try:
s = smtplib.SMTP("localhost")
s.sendmail(from_addr, to_addr, email_obj.as_string())
s.quit()
except smtplib.SMTPException as e:
raise SendEmailError("Error with SMTP: %s" % str(e))
def _send_links(self, links, lc, os, from_addr, to_addr):
"""Send links to the user.
Get the message in the proper language (according to the locale),
replace variables and send the email.
:param: links (string) the links to be sent.
:param: lc (string) the locale.
:param: os (string) the operating system.
:param: from_addr (string) the address of the sender.
:param: to_addr (string) the address of the recipient.
"""
# obtain the content in the proper language and send it
try:
links_subject = self._get_msg('links_subject', 'en')
links_msg = self._get_msg('links_msg', 'en')
links_msg = links_msg % (OS[os], links)
self._send_email(
from_addr,
to_addr,
links_subject,
links_msg,
None
)
except ConfigError as e:
raise InternalError("Error while getting message %s" % str(e))
except SendEmailError as e:
raise InternalError("Error while sending links message")
def _send_mirrors(self, lc, from_addr, to_addr):
"""Send mirrors message.
Get the message in the proper language (according to the locale),
replace variables (if any) and send the email.
:param: lc (string) the locale.
:param: from_addr (string) the address of the sender.
:param: to_addr (string) the address of the recipient.
"""
# obtain the content in the proper language and send it
try:
mirrors_subject = self._get_msg('mirrors_subject', lc)
mirrors_msg = self._get_msg('mirrors_msg', lc)
self._send_email(
from_addr, to_addr, mirrors_subject, mirrors_msg, self.mirrors
)
except ConfigError as e:
raise InternalError("Error while getting message %s" % str(e))
except SendEmailError as e:
raise InternalError("Error while sending mirrors message")
def _send_help(self, lc, from_addr, to_addr):
"""Send help message.
Get the message in the proper language (according to the locale),
replace variables (if any) and send the email.
:param: lc (string) the locale.
:param: from_addr (string) the address of the sender.
:param: to_addr (string) the address of the recipient.
"""
# obtain the content in the proper language and send it
try:
help_subject = self._get_msg('help_subject', lc)
help_msg = self._get_msg('help_msg', lc)
self._send_email(from_addr, to_addr, help_subject, help_msg, None)
except ConfigError as e:
raise InternalError("Error while getting message %s" % str(e))
except SendEmailError as e:
raise InternalError("Error while sending help message")
def process_email(self, raw_msg):
"""Process the email received.
Create an email object from the string received. The processing
flow is as following:
- check for blacklisted address.
- parse the email.
- check the type of request.
- send reply.
:param: raw_msg (string) the email received.
:raise: InternalError if something goes wrong while asking for the
links to the Core module.
"""
self.log.debug("Processing email")
parsed_msg = email.message_from_string(raw_msg)
content = self._get_content(parsed_msg)
from_addr = parsed_msg['From']
to_addr = parsed_msg['To']
bogus_request = False
status = ''
req = None
try:
# two ways for a request to be bogus: address malformed or
# blacklisted
try:
self.log.debug("Normalizing address...")
norm_from_addr = self._get_normalized_address(from_addr)
except AddressError as e:
bogus_request = True
self.log.info('invalid; none; none')
if norm_from_addr:
anon_addr = utils.get_sha256(norm_from_addr)
if self._is_blacklisted(anon_addr):
bogus_request = True
self.log.info('blacklist; none; none')
if not bogus_request:
# try to figure out what the user is asking
self.log.debug("Request seems legit; parsing it...")
req = self._parse_email(content, to_addr)
# our address should have the locale requested
our_addr = "gettor+%s@%s" % (req['lc'], self.our_domain)
# possible options: help, links, mirrors
if req['type'] == 'help':
self.log.debug("Trying to send help...")
self.log.info('help; none; %s' % req['lc'])
# make sure we can send emails
try:
self._send_help('en', our_addr, norm_from_addr)
except SendEmailError as e:
self.log.debug("FAILED: %s" % str(e))
raise InternalError("Something's wrong with the SMTP "
"server: %s" % str(e))
elif req['type'] == 'mirrors':
self.log.debug("Trying to send the mirrors...")
self.log.info('mirrors; none; %s' % req['lc'])
# make sure we can send emails
try:
self._send_mirrors('en', our_addr, norm_from_addr)
except SendEmailError as e:
self.log.debug("FAILED: %s" % str(e))
raise SendEmailError("Something's wrong with the SMTP "
"server: %s" % str(e))
elif req['type'] == 'links':
self.log.debug("Trying to obtain the links...")
self.log.info('links; %s; %s' % (req['os'], req['lc']))
try:
links = self.core.get_links(
'SMTP', req['os'], req['lc']
)
# if core fails, we fail too
except (core.InternalError, core.ConfigError) as e:
self.log.debug("FAILED: %s" % str(e))
# something went wrong with the core
raise InternalError("Error obtaining the links")
# make sure we can send emails
self.log.debug("Trying to send the links...")
try:
self._send_links(links, req['lc'], req['os'], our_addr,
norm_from_addr)
except SendEmailError as e:
self.log.debug("FAILED: %s" % str(e))
raise SendEmailError("Something's wrong with the SMTP "
"server: %s" % str(e))
finally:
self.log.debug("Request processed")

131
gettor/utils.py
View File

@ -1,131 +0,0 @@
# -*- coding: utf-8 -*-
#
# This file is part of GetTor, a Tor Browser distribution system.
#
# :authors: Israel Leiva <ilv@riseup.net>
# see also AUTHORS file
#
# :copyright: (c) 2008-2014, The Tor Project, Inc.
# (c) 2014, Israel Leiva
#
# :license: This is Free Software. See LICENSE for license information.
import os
import re
import hashlib
"""Common utilities for GetTor modules."""
LOGGING_FORMAT = "[%(levelname)s] %(asctime)s; %(message)s"
DATE_FORMAT = "%Y-%m-%d" # %H:%M:%S
windows_regex = '^torbrowser-install-\d\.\d(\.\d)?_(\w\w)(-\w\w)?\.exe$'
linux_regex = '^tor-browser-linux(\d\d)-\d\.\d(\.\d)?_(\w\w)(-\w\w)?\.tar\.xz$'
osx_regex = '^TorBrowser-\d\.\d(\.\d)?-osx\d\d_(\w\w)(-\w\w)?\.dmg$'
def get_logging_format():
"""Get the logging format.
:return: (string) the logging format.
"""
return LOGGING_FORMAT
def get_date_format():
"""Get the date format for logging.
:return: (string) the date format for logging.
"""
return DATE_FORMAT
def get_sha256(string):
"""Get sha256 of a string.
:param: (string) the string to be hashed.
:return: (string) the sha256 of string.
"""
return str(hashlib.sha256(string).hexdigest())
def get_bundle_info(filename, osys=None):
"""Get the os, arch and lc from a bundle string.
:param: file (string) the name of the file.
:param: osys (string) the OS.
:raise: ValueError if the bundle doesn't have a valid bundle format.
:return: (list) the os, arch and lc.
"""
m_windows = re.search(windows_regex, filename)
m_linux = re.search(linux_regex, filename)
m_osx = re.search(osx_regex, filename)
if m_windows:
return 'windows', '32/64', m_windows.group(2)
elif m_linux:
return 'linux', m_linux.group(1), m_linux.group(3)
elif m_osx:
return 'osx', '64', m_osx.group(2)
else:
raise ValueError("Invalid bundle format %s" % file)
def valid_format(filename, osys=None):
"""Check for valid bundle format
Check if the given file has a valid bundle format
(e.g. tor-browser-linux32-3.6.2_es-ES.tar.xz)
:param: file (string) the name of the file.
:return: (boolean) true if the bundle format is valid, false otherwise.
"""
m_windows = re.search(windows_regex, filename)
m_linux = re.search(linux_regex, filename)
m_osx = re.search(osx_regex, filename)
if any((m_windows, m_linux, m_osx)):
return True
else:
return False
def get_file_sha256(file):
"""Get the sha256 of a file.
:param: file (string) the path of the file.
:return: (string) the sha256 hash.
"""
# as seen on the internetz
BLOCKSIZE = 65536
hasher = hashlib.sha256()
with open(file, 'rb') as afile:
buf = afile.read(BLOCKSIZE)
while len(buf) > 0:
hasher.update(buf)
buf = afile.read(BLOCKSIZE)
return hasher.hexdigest()
def find_files_to_upload(upload_dir):
"""
Find the files which are named correctly and have a .asc file
"""
files = []
for name in os.listdir(upload_dir):
asc_file = os.path.join(upload_dir, "{}.asc".format(name))
if valid_format(name) and os.path.isfile(asc_file):
files.extend([name, "{}.asc".format(name)])
return files

2
gettor/blacklist.py → gettor/utils/blacklist.py
View File

@ -15,7 +15,7 @@ import time
import logging
import sqlite3
import datetime
import ConfigParser
import configparser
import db
import utils

58
gettor/utils/commons.py Normal file
View File

@ -0,0 +1,58 @@
# -*- coding: utf-8 -*-
#
# This file is part of GetTor, a Tor Browser distribution system.
#
# :authors: isra <ilv@torproject.org>
# see also AUTHORS file
#
# :copyright: (c) 2008-2014, The Tor Project, Inc.
# (c) 2014-2018, Israel Leiva
#
# :license: This is Free Software. See LICENSE for license information.
from twisted.logger import Logger
from twisted.enterprise import adbapi
from twisted.application import internet
# Define an application logger
log = Logger(namespace="gettor")
class BaseService(internet.TimerService):
"""
Base service for Email and Sendmail. It extends the TimerService.
"""
def __init__(self, name, step, instance, *args, **kwargs):
"""
Constructor.
:param name (str): name of the service (just for logging purposes).
:param step (float): time interval for TimerService, in seconds.
:param instance (object): instance of Email, Sendmail classes
"""
log.info("SERVICE:: Initializing {} service.".format(name))
self.name = name
self.instance = instance
log.debug("SERVICE:: Initializing TimerService.")
internet.TimerService.__init__(
self, step, self.instance.get_new, **kwargs
)
def startService(self):
"""
Starts the service. Overridden from parent class to add extra logging
information.
"""
log.info("SERVICE:: Starting {} service.".format(self.name))
internet.TimerService.startService(self)
log.info("SERVICE:: Service started.")
def stopService(self):
"""
Stop the service. Overridden from parent class to close connection to
database, shutdown the service and add extra logging information.
"""
log.info("SERVICE:: Stopping {} service.".format(self.name))
internet.TimerService.stopService(self)
log.info("SERVICE:: Service stopped.")

7
gettor/db.py → gettor/utils/db.py
View File

@ -39,14 +39,11 @@ class DB(object):
"""
def __init__(self, dbname):
"""Create a new db object.
:param: dbname (string) the path of the database.
"""Create a new db object.
:param: dbname (string) the path of the database.
"""
self.dbname = dbname
def connect(self):
""" """
try:

30
gettor/utils/options.py Normal file
View File

@ -0,0 +1,30 @@
# -*- coding: utf-8 -*-
"""
This file is part of GetTor, a service providing alternative methods to download
the Tor Browser.
:authors: Hiro <hiro@torproject.org>
parser = argparse.ArgumentParser(formatter_class=lambda prog: argparse.HelpFormatter(prog,max_help_position=28))
parser.add_argument('--config', metavar='config', please also see AUTHORS file
:copyright: (c) 2008-2014, The Tor Project, Inc.
(c) 2014, all entities within the AUTHORS file
:license: see included LICENSE for information
"""
import argparse
from .settings import Settings
from . import strings
def load_settings(config=None):
"""
Loading settings, optionally from a custom config json file.
"""
settings = Settings(config)
settings.load()
return settings
def parse_settings():
strings.load_strings("en")
return load_settings(config=False)

62
gettor/utils/settings.py Normal file
View File

@ -0,0 +1,62 @@
# -*- coding: utf-8 -*-
"""
This file is part of GetTor, a service providing alternative methods to download
the Tor Browser.
:authors: Hiro <hiro@torproject.org>
please also see AUTHORS file
:copyright: (c) 2008-2014, The Tor Project, Inc.
(c) 2014, all entities within the AUTHORS file
:license: see included LICENSE for information
"""
import json
import os
import platform
import locale
from . import strings
class Settings(object):
"""
This class stores all of the settings for GetTor
"""
def __init__(self, config=False):
# Default config
self.filename = self.build_filename()
# If a readable config file was provided, use that instead
if config:
if os.path.isfile(config):
self.filename = config
# Dictionary of available languages,
# mapped to the language name, in that language
self._available_locales = strings.get_locales()
self._version = strings.get_version()
self._settings = {}
def build_filename(self):
"""
Returns the path of the settings file.
"""
return strings.get_resource_path('gettor.conf.json', strings.find_run_dir())
def load(self):
"""
Load the settings from file.
"""
# If the settings file exists, load it
if os.path.exists(self.filename):
try:
with open(self.filename, 'r') as f:
self._settings = json.load(f)
except:
pass
def get(self, key):
return self._settings[key]

122
gettor/utils/strings.py Normal file
View File

@ -0,0 +1,122 @@
# -*- coding: utf-8 -*-
"""
This file is part of GetTor, a service providing alternative methods to download
the Tor Browser.
:authors: Hiro <hiro@torproject.org>
please also see AUTHORS file
:copyright: (c) 2008-2014, The Tor Project, Inc.
(c) 2014, all entities within the AUTHORS file
:license: see included LICENSE for information
"""
import json
import locale
import os
import inspect
strings = {}
translations = {}
_rundir = None
def setRundir(path):
"""Set the absolute path to the runtime directory.
See :meth:`BaseOptions.postOptions`.
:param string path: The path to set.
"""
global _rundir
_rundir = path
def getRundir():
"""Get the absolute path to the runtime directory.
:rtype: string
:returns: The path to the config file.
"""
return _rundir
def find_run_dir(rundir=None):
"""Get the absolute path to the runtime directory.
:rtype: string
:returns: The path to the config file.
"""
gRundir = getRundir()
if gRundir is None:
if rundir is not None:
gRundir = os.path.abspath(os.path.expanduser(rundir))
else:
gRundir = os.getcwd()
setRundir(gRundir)
if not os.path.isdir(gRundir): # pragma: no cover
raise usage.UsageError(
"Could not change to runtime directory: `%s'" % gRundir)
return gRundir
def get_resource_path(filename, path):
"""
Returns the absolute path of a resource
"""
rundir = find_run_dir()
prefix = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))), path)
prefix = os.path.join(rundir, prefix)
if not os.path.exists(prefix):
prefix = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(prefix)))), path)
return os.path.join(prefix, filename)
def get_version():
# The current version
version = ""
with open(get_resource_path('version.txt', '../share')) as f:
version = f.read().strip()
return version
def get_locales():
filename = get_resource_path("available_locales.json", '../share/locale')
locales = {}
with open(filename, encoding='utf-8') as f:
locales = json.load(f)
return locales
def load_strings(current_locale):
"""
Loads translated strings and fallback to English
if the translation does not exist.
"""
global strings, translations
# Load all translations
translations = {}
available_locales = get_locales()
for locale in available_locales:
filename = get_resource_path("{}.json".format(locale), '../share/locale')
with open(filename, encoding='utf-8') as f:
translations[locale] = json.load(f)
# Build strings
default_locale = 'en'
strings = {}
for s in translations[default_locale]:
if s in translations[current_locale] and translations[current_locale][s] != "":
strings[s] = translations[current_locale][s]
else:
strings[s] = translations[default_locale][s]
def translated(k):
"""
Returns a translated string.
"""
return strings[k]
_ = translated

2
gettor/http.py → gettor/web/http.py
View File

@ -15,7 +15,7 @@ import re
import json
import codecs
import urllib2
import ConfigParser
import configparser
from time import gmtime, strftime

154
scripts/create_db Executable file
View File

@ -0,0 +1,154 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# This file is part of GetTor, a Tor Browser distribution system.
#
# :authors: isra <ilv@torproject.org>
# see also AUTHORS file
#
# :license: This is Free Software. See LICENSE for license information.
import os
import sys
import sqlite3
import argparse
import subprocess
from shutil import move
def print_header():
header = """
__ __
/\ \__ /\ \__
__ __\ \ ,_\\\ \ ,_\ ____ _ __
/'_ `\ /'__`\ \ \/ \ \ \/ / __ `\/\`'__\
/\ \L\ \/\ __/\ \ \_ \ \ \ /\ \L\ \ \ \/
\ \____ \ \____\\\__\ \ \ \__\ \_____/\ \_\
\/___L\ \/____/ \/__/ \/__/\/___/ \/_/
/\_____/
\_/___/
"""
print("")
print("@"*100)
print("@"*100)
print(header)
print("@"*100)
print("")
def print_footer():
print("")
print("@"*100)
print("@"*100)
print("")
def main():
parser = argparse.ArgumentParser(
description="Tool to create the gettor SQLite database."
)
parser.add_argument(
"-f", "--filename", default="gettor.db", metavar="gettor.db",
help="Database filename."
)
parser.add_argument(
"-n", "--new", action="store_true",
help="Create new database file.")
parser.add_argument(
"-o", "--overwrite", action="store_true",
help="Overwrite existing database file."
)
parser.add_argument(
"-c", "--clear", action="store_true",
help="Clear database."
)
args = parser.parse_args()
abs_filename = os.path.abspath(args.filename)
if not abs_filename:
print("Missing database filename.")
elif args.new and not args.overwrite and os.path.isfile(abs_filename):
print("Database file already exists. Use -o to overwrite.")
elif args.new:
conn = sqlite3.connect(abs_filename)
with conn:
c = conn.cursor()
c.execute("DROP TABLE IF EXISTS requests")
c.execute("DROP TABLE IF EXISTS links")
c.execute("DROP TABLE IF EXISTS stats")
c.execute(
"CREATE TABLE requests(id TEXT, command TEXT, platform TEXT,"
" service TEXT, date TEXT, status TEXT)"
)
c.execute(
"CREATE TABLE links(link TEXT, platform TEXT, arch TEXT,"
" version TEXT, provider TEXT, status TEXT)"
)
c.execute(
"CREATE TABLE stats(num_requests NUMBER, platform TEXT, "
"command TEXT, service TEXT, date TEXT)"
)
print("Database {} created.".format(abs_filename))
elif args.clear:
print("Shredding database file.")
if not abs_filename:
print("Database file does not exist.")
else:
move(abs_filename, "{}.tmp".format(abs_filename))
if os.path.isfile("{}.tmp".format(abs_filename)):
print("Database moved to {}.tmp.".format(
abs_filename
))
conn = sqlite3.connect(abs_filename)
with conn:
c = conn.cursor()
c.execute("DROP TABLE IF EXISTS requests")
c.execute("DROP TABLE IF EXISTS links")
c.execute("DROP TABLE IF EXISTS stats")
c.execute(
"CREATE TABLE requests(id TEXT, command TEXT, "
"platform TEXT, service TEXT, date TEXT, status TEXT,"
"PRIMARY KEY(id, date))"
)
c.execute(
"CREATE TABLE links(link TEXT, platform TEXT, "
"arch TEXT, version TEXT, provider TEXT, status TEXT,"
"PRIMARY KEY(platform, arch, version, provider, status))"
)
c.execute(
"CREATE TABLE stats(date TEXT PRIMARY KEY, "
"num_requests INTEGER, platform TEXT, command TEXT, "
"service TEXT)"
)
print("New database {} created.".format(abs_filename))
cmd = subprocess.call(
[
"shred", "-z", "-n", "100", "-u", "{}.tmp".format(
abs_filename
)
]
)
if cmd:
sys.exit("Error while shredding database file.")
else:
print("Database file {}.tmp shredded.".format(abs_filename))
else:
print("Could not create temporary database file.")
if __name__ == "__main__":
print_header()
main()
print_footer

30
scripts/gettor Normal file
View File

@ -0,0 +1,30 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
This file is part of GetTor, a service providing alternative methods to download
the Tor Browser.
:authors: Hiro <hiro@torproject.org>
please also see AUTHORS file
:copyright: (c) 2008-2014, The Tor Project, Inc.
(c) 2014, all entities within the AUTHORS file
:license: see included LICENSE for information
"""
from __future__ import print_function
import os.path
import sys
from os import getcwd
from twisted.application import service
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from gettor.main import run
gettor = service.MultiService()
application = service.Application("gettor")
run(gettor, application)

257
setup.py
View File

@ -0,0 +1,257 @@
# -*- coding: utf-8 -*-
"""
This file is part of GetTor, a service providing alternative methods to download
the Tor Browser.
:authors: Hiro <hiro@torproject.org>
please also see AUTHORS file
:copyright: (c) 2008-2014, The Tor Project, Inc.
(c) 2014, all entities within the AUTHORS file
:license: see included LICENSE for information
"""
from __future__ import print_function
import os
import setuptools
import sys
def get_cmdclass():
"""Get our cmdclass dictionary for use in setuptool.setup().
This must be done outside the call to setuptools.setup() because we need
to add our own classes to the cmdclass dictionary, and then update that
dictionary with the one returned from versioneer.get_cmdclass().
"""
cmdclass = {'test': Trial,
'compile_catalog': compile_catalog,
'extract_messages': extract_messages,
'init_catalog': init_catalog,
'update_catalog': update_catalog}
cmdclass.update(versioneer.get_cmdclass())
return cmdclass
def get_requirements():
"""Extract the list of requirements from our requirements.txt.
:rtype: 2-tuple
:returns: Two lists, the first is a list of requirements in the form of
pkgname==version. The second is a list of URIs or VCS checkout strings
which specify the dependency links for obtaining a copy of the
requirement.
"""
requirements_file = os.path.join(os.getcwd(), 'requirements.txt')
requirements = []
links=[]
try:
with open(requirements_file) as reqfile:
for line in reqfile.readlines():
line = line.strip()
if line.startswith('#'):
continue
if line.startswith(('git+', 'hg+', 'svn+')):
line = line[line.index('+') + 1:]
if line.startswith(
('https://', 'git://', 'hg://', 'svn://')):
links.append(line)
else:
requirements.append(line)
except (IOError, OSError) as error:
print(error)
return requirements, links
def get_template_files():
"""Return the paths to any web resource files to include in the package.
:rtype: list
:returns: Any files in :attr:`repo_templates` which match one of the glob
patterns in :ivar:`include_patterns`.
"""
include_patterns = ['*.html',
'*.txt',
'*.asc',
'share/*.png',
'share/*.svg',
'share/css/*.css',
'share/fonts/*.woff',
'share/fonts/*.ttf',
'share/fonts/*.svg',
'share/fonts/*.eot',
'share/images/*.svg']
template_files = []
for include_pattern in include_patterns:
pattern = os.path.join(repo_templates, include_pattern)
matches = glob(pattern)
template_files.extend(matches)
return template_files
def get_data_files(filesonly=False):
"""Return any hard-coded data_files which should be distributed.
This is necessary so that both the distutils-derived :class:`installData`
class and the setuptools ``data_files`` parameter include the same files.
Call this function with ``filesonly=True`` to get a list of files suitable
for giving to the ``package_data`` parameter in ``setuptools.setup()``.
Or, call it with ``filesonly=False`` (the default) to get a list which is
suitable for using as ``distutils.command.install_data.data_files``.
:param bool filesonly: If true, only return the locations of the files to
install, not the directories to install them into.
:rtype: list
:returns: If ``filesonly``, returns a list of file paths. Otherwise,
returns a list of 2-tuples containing: one, the directory to install
to, and two, the files to install to that directory.
"""
data_files = []
doc_files = ['README', 'TODO', 'LICENSE', 'requirements.txt']
lang_dirs, lang_files = get_supported_langs()
template_files = get_template_files()
if filesonly:
data_files.extend(doc_files)
for lst in lang_files, template_files:
for filename in lst:
if filename.startswith(pkgpath):
# The +1 gets rid of the '/' at the beginning:
filename = filename[len(pkgpath) + 1:]
data_files.append(filename)
else:
data_files.append((install_docs, doc_files))
for ldir, lfile in zip(lang_dirs, lang_files):
data_files.append((ldir, [lfile,]))
#[sys.stdout.write("Added data_file '%s'\n" % x) for x in data_files]
return data_files
class Trial(setuptools.Command):
"""Twisted Trial setuptools command.
Based on the setuptools Trial command in Zooko's Tahoe-LAFS, as well as
https://github.com/simplegeo/setuptools-trial/ (which is also based on the
Tahoe-LAFS code).
Pieces of the original implementation of this 'test' command (that is, for
the original pyunit-based gettor tests which, a long time ago, in a
galaxy far far away, lived in gettor.Tests) were based on setup.py from
Nick Mathewson's mixminion, which was based on the setup.py from Zooko's
pyutil package, which was in turn based on
http://mail.python.org/pipermail/distutils-sig/2002-January/002714.html.
Crusty, old-ass Python, like hella wut.
"""
description = "Run Twisted Trial-based tests."
user_options = [
('debug', 'b', ("Run tests in a debugger. If that debugger is pdb, will "
"load '.pdbrc' from current directory if it exists.")),
('debug-stacktraces', 'B', "Report Deferred creation and callback stack traces"),
('debugger=', None, ("The fully qualified name of a debugger to use if "
"--debug is passed (default: pdb)")),
('disablegc', None, "Disable the garbage collector"),
('force-gc', None, "Have Trial run gc.collect() before and after each test case"),
('jobs=', 'j', "Number of local workers to run, a strictly positive integer"),
('profile', None, "Run tests under the Python profiler"),
('random=', 'Z', "Run tests in random order using the specified seed"),
('reactor=', 'r', "Which reactor to use"),
('reporter=', None, "Customize Trial's output with a reporter plugin"),
('rterrors', 'e', "Realtime errors: print out tracebacks as soon as they occur"),
('spew', None, "Print an insanely verbose log of everything that happens"),
('testmodule=', None, "Filename to grep for test cases (-*- test-case-name)"),
('tbformat=', None, ("Specify the format to display tracebacks with. Valid "
"formats are 'plain', 'emacs', and 'cgitb' which uses "
"the nicely verbose stdlib cgitb.text function")),
('unclean-warnings', None, "Turn dirty reactor errors into warnings"),
('until-failure', 'u', "Repeat a test (specified by -s) until it fails."),
('without-module=\'python3 setup.py test\'', None, ("Fake the lack of the specified modules, separated "
"with commas")),
]
boolean_options = ['debug', 'debug-stacktraces', 'disablegc', 'force-gc',
'profile', 'rterrors', 'spew', 'unclean-warnings',
'until-failure']
def initialize_options(self):
self.debug = None
self.debug_stacktraces = None
self.debugger = None
self.disablegc = None
self.force_gc = None
self.jobs = None
self.profile = None
self.random = None
self.reactor = None
self.reporter = None
self.rterrors = None
self.spew = None
self.testmodule = None
self.tbformat = None
self.unclean_warnings = None
self.until_failure = None
self.without_module = None
def finalize_options(self):
build = self.get_finalized_command('build')
self.build_purelib = build.build_purelib
self.build_platlib = build.build_platlib
def run(self):
self.run_command('build')
old_path = sys.path[:]
sys.path[0:0] = [self.build_purelib, self.build_platlib]
result = 1
try:
result = self.run_tests()
finally:
sys.path = old_path
raise SystemExit(result)
def run_tests(self):
# We do the import from Twisted inside the function instead of the top
# of the file because since Twisted is a setup_requires, we can't
# assume that Twisted will be installed on the user's system prior, so
# if we don't do the import here, then importing from this plugin will
# fail.
from twisted.scripts import trial
if not self.testmodule:
self.testmodule = "test"
# Handle parsing the trial options passed through the setuptools
# trial command.
cmd_options = []
for opt in self.boolean_options:
if getattr(self, opt.replace('-', '_'), None):
cmd_options.append('--%s' % opt)
for opt in ('debugger', 'jobs', 'random', 'reactor', 'reporter',
'testmodule', 'tbformat', 'without-module'):
value = getattr(self, opt.replace('-', '_'), None)
if value is not None:
cmd_options.extend(['--%s' % opt, value])
config = trial.Options()
config.parseOptions(cmd_options)
config['tests'] = [self.testmodule,]
trial._initialDebugSetup(config)
trialRunner = trial._makeRunner(config)
suite = trial._getSuite(config)
# run the tests
if self.until_failure:
test_result = trialRunner.runUntilFailure(suite)
else:
test_result = trialRunner.run(suite)
if test_result.wasSuccessful():
return 0 # success
return 1 # failure

0
css/bootstrap.min.css → share/css/bootstrap.min.css vendored
View File

0
css/gettor.css → share/css/gettor.css
View File

3
share/locale/available_locales.json Normal file
View File

@ -0,0 +1,3 @@
{
"en": "English"
}

12
share/locale/en.json
View File

@ -0,0 +1,12 @@
{
"help_debug": "Log application errors to stdout",
"help_config": "Custom config file location (optional)",
"smtp_links_subject": "[GetTor] Links for your request",
"smtp_mirrors_subject": "[GetTor] Mirrors",
"smtp_help_subject": "[GetTor] Help",
"smtp_unsupported_locale_subject": "[GetTor] Unsupported locale",
"smtp_unsupported_locale_msg": "The locale you requested '%s' is not supported.",
"smtp_vlinks_msg": "You requested Tor Browser for %s.\n \nYou will need only one of the links below to download the bundle. If a link does not work for you, try the next one.\n \n%s\n \n \n--\nGetTor",
"smtp_mirrors_msg": "Hi! this is the GetTor robot.\n \nThank you for your request. Attached to this email you will find\nan updated list of mirrors of Tor Project's website.",
"smtp_help_msg": "Hi! This is the GetTor robot. I am here to help you download the\nlatest version of Tor Browser.\n \nPlease reply to this message with one of the options below:\n \nwindows\nlinux\nosx\nmirrors\n \nI will then send you the download instructions.\n \nIf you are unsure, just send a blank reply to this message."
}

1
share/version.txt Normal file
View File

@ -0,0 +1 @@
2.0.dev1

0
gettor.conf → test/__init__.py
View File

27
test/test_email_service.py Normal file
View File

@ -0,0 +1,27 @@
# -*- coding: utf-8 -*-
"""
This file is part of GetTor, a service providing alternative methods to download
the Tor Browser.
:authors: Hiro <hiro@torproject.org>
please also see AUTHORS file
:copyright: (c) 2008-2014, The Tor Project, Inc.
(c) 2014, all entities within the AUTHORS file
:license: see included LICENSE for information
"""
from __future__ import print_function
from .gettor.utils import options
class EmailServiceTests(unittest.TestCase):
# Fail any tests which take longer than 15 seconds.
timeout = 15
def setUp(self):
self.setings = options.parse_settings()
def test_SendMail(self):
sendmail = Sendmail(self.settings)
mail = Sendmail.sendmail("gettor@torproject.org", "Hello", "This is a test.")
print(email)
self.assertEqual(mail, True)