torproject/gettor
1
0
Fork 0
mirror of https://github.com/torproject/gettor.git synced 2024-11-23 09:49:41 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add method to close dbpool and call it

Browse Source 
We were leaving connections to the database open, which causes some
calls to hang. This adds destructors to the SQLite3 class and the
classes that use it, and refactors some code to make one database
connection per class object. This also makes sure that the destructors
for objects that use databases are actually called.
This commit is contained in:
Cecylia Bocovich 2020-02-19 12:15:58 -05:00
parent c6cfd91241
commit c413a8ebda
8 changed files with 39 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
gettor/parse/email.py
View File

@ -56,6 +56,10 @@ class EmailParser(object):
self.to_addr = to_addr
self.locales = []
self.platforms = self.settings.get("platforms")
self.conn = SQLite3(self.settings.get("dbname"))
def __del__(self):
del self.conn
def normalize(self, msg):
# Normalization will convert <Alice Wonderland> alice@wonderland.net
@ -164,10 +168,8 @@ class EmailParser(object):
@defer.inlineCallbacks
def get_locales(self):
dbname = self.settings.get("dbname")
conn = SQLite3(dbname)
locales = yield conn.get_locales()
locales = yield self.conn.get_locales()
for l in locales:
self.locales.append(l[0])
@ -235,7 +237,6 @@ class EmailParser(object):
now_str = datetime.now().strftime("%Y%m%d%H%M%S")
dbname = self.settings.get("dbname")
test_hid = self.settings.get("test_hid")
conn = SQLite3(dbname)
if request["command"]:
@ -247,7 +248,7 @@ class EmailParser(object):
system="email parser"
)
num_requests = yield conn.get_num_requests(
num_requests = yield self.conn.get_num_requests(
id=hid, service=request_service
)
@ -262,7 +263,7 @@ class EmailParser(object):
), system="email parser"
)
else:
conn.new_request(
self.conn.new_request(
id=request['id'],
command=request['command'],
platform=request['platform'],

11
gettor/parse/twitter.py
View File

@ -35,7 +35,10 @@ class TwitterParser(object):
"""
self.settings = settings
self.twitter_id = twitter_id
self.conn = SQLite3(self.settings.get("dbname"))
def __del__(self):
del self.conn
def build_request(self, msg_text, twitter_id, languages, platforms):
@ -110,16 +113,14 @@ class TwitterParser(object):
if request["command"]:
now_str = datetime.now().strftime("%Y%m%d%H%M%S")
dbname = self.settings.get("dbname")
conn = SQLite3(dbname)
hid = hashlib.sha256(str(request['id']).encode('utf-8'))
# check limits first
num_requests = yield conn.get_num_requests(
num_requests = yield self.conn.get_num_requests(
id=hid.hexdigest(), service=request['service']
)
num_requests += yield conn.get_num_requests(
num_requests += yield self.conn.get_num_requests(
id=str(request['id']), service=request['service']
)
@ -130,7 +131,7 @@ class TwitterParser(object):
), system="twitter parser"
)
else:
conn.new_request(
self.conn.new_request(
id=str(request['id']),
command=request['command'],
platform=request['platform'],

2
gettor/services/email/sendmail.py
View File

@ -45,6 +45,8 @@ class Sendmail(object):
dbname = self.settings.get("dbname")
self.conn = DB(dbname)
def __del__(self):
del self.conn
def get_interval(self):
"""

3
gettor/services/twitter/twitterdm.py
View File

@ -41,6 +41,8 @@ class Twitterdm(object):
self.twitter = Twitter(settings)
self.conn = DB(dbname)
def __del__(self):
del self.conn
def get_interval(self):
"""
@ -115,6 +117,7 @@ class Twitterdm(object):
yield defer.maybeDeferred(
tp.parse, e['message_create']['message_data']['text'], message_id
).addCallback(tp.parse_callback).addErrback(tp.parse_errback)
del tp
# Manage help and links messages separately
help_requests = yield self.conn.get_requests(

3
gettor/utils/db.py
View File

@ -24,6 +24,9 @@ class SQLite3(object):
"sqlite3", dbname, check_same_thread=False
)
def __del__(self):
self.dbpool.close()
def query_callback(self, results=None):
"""
Query callback

1
scripts/process_email
View File

@ -41,6 +41,7 @@ def process_email(message):
log.err("DKIM error: {}".format(e), system="process email")
reactor.stop()
del ep
reactor.stop()
def main():

4
tests/test_db.py
View File

@ -2,8 +2,6 @@
import pytest
import pytest_twisted
from twisted.trial import unittest
from twisted.internet import defer, reactor
from twisted.internet import task
from . import conftests
@ -19,7 +17,7 @@ class DatabaseTests(unittest.TestCase):
def tearDown(self):
print("tearDown()")
return self.conn.dbpool.close()
del self.conn
@pytest_twisted.inlineCallbacks
def test_stored_locales(self):

16
tests/test_email_service.py
View File

@ -1,5 +1,6 @@
#!/usr/bin/env python3
import pytest
import pytest_twisted
import hashlib
from datetime import datetime
from twisted.trial import unittest
@ -31,6 +32,7 @@ class EmailServiceTests(unittest.TestCase):
def tearDown(self):
print("tearDown()")
del self.sm_client
def test_get_interval(self):
self.assertEqual(self.settings.get("sendmail_interval"), self.sm_client.get_interval())
@ -39,6 +41,7 @@ class EmailServiceTests(unittest.TestCase):
ep = conftests.EmailParser(self.settings, "gettor@torproject.org")
request = ep.parse("From: \"silvia [hiro]\" <hiro@torproject.org>\n Subject: help\n Reply-To: hiro@torproject.org \nTo: gettor@torproject.org")
self.assertEqual(request["command"], "help")
del ep
def test_normalize_msg(self):
ep = conftests.EmailParser(self.settings, "gettor@torproject.org")
@ -46,6 +49,7 @@ class EmailServiceTests(unittest.TestCase):
msg = conftests.message_from_string(msg_str)
request = ep.normalize(msg)
self.assertEqual(request, ('silvia [hiro]', 'hiro@torproject.org', '', 'gettor@torproject.org'))
del ep
def test_validate_msg(self):
ep = conftests.EmailParser(self.settings, "gettor@torproject.org")
@ -53,6 +57,7 @@ class EmailServiceTests(unittest.TestCase):
msg = conftests.message_from_string(msg_str)
request = ep.validate("hiro@torproject.org", msg)
assert request
del ep
def test_dkim_verify(self):
ep = conftests.EmailParser(self.settings, "gettor@torproject.org")
@ -60,6 +65,7 @@ class EmailServiceTests(unittest.TestCase):
msg = conftests.message_from_string(msg_str)
request = ep.dkim_verify(msg, "hiro@torproject.org")
assert request
del ep
def test_build_request(self):
ep = conftests.EmailParser(self.settings, "gettor@torproject.org")
@ -70,6 +76,7 @@ class EmailServiceTests(unittest.TestCase):
self.assertEqual(request["command"], "links")
self.assertEqual(request["platform"], "osx")
self.assertEqual(request["language"], "es")
del ep
def test_too_many_request_exclude(self):
ep = conftests.EmailParser(self.settings, "gettor@torproject.org")
@ -79,6 +86,7 @@ class EmailServiceTests(unittest.TestCase):
check = ep.too_many_requests(hid, self.settings.get("test_hid"), num_requests, limit)
self.assertEqual(hid, self.settings.get("test_hid"))
self.assertEqual(check, False)
del ep
def test_language_email_parser(self):
ep = conftests.EmailParser(self.settings, "gettor@torproject.org")
@ -137,6 +145,7 @@ class EmailServiceTests(unittest.TestCase):
"Subject: linux es-AR\r\n Reply-To: hiro@torproject.org \nTo:"
"gettor@torproject.org\n linux es")
self.assertEqual(request["language"], "es-AR")
del ep
def test_sent_links_message(self):
ep = self.sm_client
@ -159,6 +168,13 @@ class EmailServiceTests(unittest.TestCase):
help_msg = ep.build_help_body_message()
assert "This is how you can request a tor browser bundle link" in help_msg
@pytest_twisted.inlineCallbacks
def test_get_locales(self):
ep = conftests.EmailParser(self.settings, "gettor@torproject.org")
yield ep.get_locales().addErrback(ep.parse_errback)
assert "en-US" in ep.locales
del ep
if __name__ == "__main__":
unittest.main()