Magic-Mirror/mitmproxy
1
0
Fork 0
mirror of https://github.com/mitmproxy/mitmproxy.git synced 2024-11-23 13:19:48 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Remove odict

Browse Source 
- Adds default implementations for _kconv and _reduce_values to MultiDict.
Without these, operations fail in really, really non-obvious ways.
- Replace the remaining few instances of ODict

Fixes #1159
This commit is contained in:
Aldo Cortesi 2016-06-09 13:28:43 +12:00
parent 90cb84b536
commit c421c41307
8 changed files with 26 additions and 334 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
docs/dev/addingviews.html
View File

@ -18,22 +18,22 @@ __prompt__, and __content\_types__ and a function named __\_\_call\_\___.
Adding a new content viewer to parse a data type is as simple as writing a new
View class. Your new content viewer View class should have the same properties
as the other View classes: __name__, __prompt__, and __content\_types__ and a
__\_\_call\_\___ function to parse the content of the request/response.
__\_\_call\_\___ function to parse the content of the request/response.
* The __name__ property should be a string describing the contents and new content viewer;
* The __name__ property should be a string describing the contents and new content viewer;
* The __prompt__ property should be a two item tuple:
- __1__: A string that will be used to display the new content viewer's type; and
- __2__: A one character string that will be the hotkey used to select the new content viewer from the Flow View screen;
- __2__: A one character string that will be the hotkey used to select the new content viewer from the Flow View screen;
* The __content\_types__ property should be a list of strings of HTTP Content\-Types that the new content viewer can parse.
* The __content\_types__ property should be a list of strings of HTTP Content\-Types that the new content viewer can parse.
* Note that mitmproxy will use the content\_types to try and heuristically show a friendly view of content and that you can override the built-in views by populating content\_types with values for content\_types that are already parsed -- e.g. "image/png".
After defining the __name__, __prompt__, and __content\_types__ properties of
the class, you should write the __\_\_call\_\___ function, which will parse the
request/response data and provide a friendly view of the data. The
__\_\_call\_\___ function should take the following arguments: __self__,
__hdrs__, __content__, __limit__; __hdrs__ is a ODictCaseless object containing
__hdrs__, __content__, __limit__; __hdrs__ is a MultiDict object containing
the headers of the request/response; __content__ is the content of the
request/response, and __limit__ is an integer representing the amount of data
to display in the view window.
@ -46,7 +46,7 @@ Alternatively, you can display content as a series of key-value pairs; to do
so, prepare a list of lists, where each list item is a two item list -- a key
that describes the data, and then the data itself; after preparing the list of
lists, use the __common.format\_keyvals__ function on it to prepare it as text
for display.
for display.
If the new content viewer fails or throws an exception, mitmproxy will default
to a __raw__ view.

8
mitmproxy/contentviews.py
View File

@ -33,7 +33,7 @@ from mitmproxy.contrib import jsbeautifier
from mitmproxy.contrib.wbxml import ASCommandResponse
from netlib import encoding
from netlib import http
from netlib import odict
from netlib import multidict
from netlib.http import url
from netlib import strutils
@ -277,7 +277,7 @@ class ViewURLEncoded(View):
def __call__(self, data, **metadata):
d = url.decode(data)
return "URLEncoded form", format_dict(odict.ODict(d))
return "URLEncoded form", format_dict(multidict.MultiDict(d))
class ViewMultipart(View):
@ -288,7 +288,7 @@ class ViewMultipart(View):
@staticmethod
def _format(v):
yield [("highlight", "Form data:\n")]
for message in format_dict(odict.ODict(v)):
for message in format_dict(multidict.MultiDict(v)):
yield message
def __call__(self, data, **metadata):
@ -437,7 +437,7 @@ class ViewImage(View):
parts.append(
(str(tag), str(ex[i]))
)
fmt = format_dict(odict.ODict(parts))
fmt = format_dict(multidict.MultiDict(parts))
return "%s image" % img.format, fmt

9
netlib/http/response.py
View File

@ -73,10 +73,11 @@ class Response(message.Message):
def cookies(self):
# type: () -> multidict.MultiDictView
"""
The response cookies. A possibly empty :py:class:`~netlib.multidict.MultiDictView`, where the keys are
cookie name strings, and values are (value, attr) tuples. Value is a string, and attr is
an ODictCaseless containing cookie attributes. Within attrs, unary attributes (e.g. HTTPOnly)
are indicated by a Null value.
The response cookies. A possibly empty
:py:class:`~netlib.multidict.MultiDictView`, where the keys are cookie
name strings, and values are (value, attr) tuples. Value is a string,
and attr is an MultiDictView containing cookie attributes. Within
attrs, unary attributes (e.g. HTTPOnly) are indicated by a Null value.
Caveats:
Updating the attr

8
netlib/multidict.py
View File

@ -235,6 +235,14 @@ class MultiDict(_MultiDict):
tuple(i) for i in fields
)
@staticmethod
def _reduce_values(values):
return values[0]
@staticmethod
def _kconv(key):
return key
@six.add_metaclass(ABCMeta)
class ImmutableMultiDict(MultiDict):

170
netlib/odict.py
View File

@ -1,170 +0,0 @@
from __future__ import (absolute_import, print_function, division)
import copy
import six
from netlib import basetypes, strutils
class ODict(basetypes.Serializable):
"""
A dictionary-like object for managing ordered (key, value) data. Think
about it as a convenient interface to a list of (key, value) tuples.
"""
def __init__(self, lst=None):
self.lst = lst or []
def _kconv(self, s):
return s
def __eq__(self, other):
return self.lst == other.lst
def __ne__(self, other):
return not self.__eq__(other)
def __iter__(self):
return self.lst.__iter__()
def __getitem__(self, key):
"""
Returns a list of values matching key.
"""
key = self._kconv(key)
return [
v
for k, v in self.lst
if self._kconv(k) == key
]
def keys(self):
return list(
set(
self._kconv(k) for k, _ in self.lst
)
)
def __len__(self):
"""
Total number of (key, value) pairs.
"""
return len(self.lst)
def __setitem__(self, k, valuelist):
"""
Sets the values for key k. If there are existing values for this
key, they are cleared.
"""
if isinstance(valuelist, six.text_type) or isinstance(valuelist, six.binary_type):
raise ValueError(
"Expected list of values instead of string. "
"Example: odict[b'Host'] = [b'www.example.com']"
)
kc = self._kconv(k)
new = []
for i in self.lst:
if self._kconv(i[0]) == kc:
if valuelist:
new.append([k, valuelist.pop(0)])
else:
new.append(i)
while valuelist:
new.append([k, valuelist.pop(0)])
self.lst = new
def __delitem__(self, k):
"""
Delete all items matching k.
"""
k = self._kconv(k)
self.lst = [
i
for i in self.lst
if self._kconv(i[0]) != k
]
def __contains__(self, key):
key = self._kconv(key)
return any(
self._kconv(k) == key
for k, _ in self.lst
)
def add(self, key, value, prepend=False):
if prepend:
self.lst.insert(0, [key, value])
else:
self.lst.append([key, value])
def get(self, k, d=None):
if k in self:
return self[k]
else:
return d
def get_first(self, k, d=None):
if k in self:
return self[k][0]
else:
return d
def items(self):
return self.lst[:]
def copy(self):
"""
Returns a copy of this object.
"""
lst = copy.deepcopy(self.lst)
return self.__class__(lst)
def extend(self, other):
"""
Add the contents of other, preserving any duplicates.
"""
self.lst.extend(other.lst)
def __repr__(self):
return repr(self.lst)
def replace(self, pattern, repl, *args, **kwargs):
"""
Replaces a regular expression pattern with repl in both keys and
values.
Returns the number of replacements made.
"""
new, count = [], 0
for k, v in self.lst:
k, c = strutils.safe_subn(pattern, repl, k, *args, **kwargs)
count += c
v, c = strutils.safe_subn(pattern, repl, v, *args, **kwargs)
count += c
new.append([k, v])
self.lst = new
return count
# Implement Serializable
def get_state(self):
return [tuple(i) for i in self.lst]
def set_state(self, state):
self.lst = [list(i) for i in state]
@classmethod
def from_state(cls, state):
return cls([list(i) for i in state])
class ODictCaseless(ODict):
"""
A variant of ODict with "caseless" keys. This version _preserves_ key
case, but does not consider case when setting or getting items.
"""
def _kconv(self, s):
return s.lower()

6
test/mitmproxy/test_contentview.py
View File

@ -1,8 +1,8 @@
from mitmproxy.exceptions import ContentViewException
from netlib.http import Headers
from netlib.odict import ODict
from netlib import encoding
from netlib.http import url
from netlib import multidict
import mitmproxy.contentviews as cv
from . import tutils
@ -55,7 +55,7 @@ class TestContentView:
f = v(
"",
headers=Headers(),
query=ODict([("foo", "bar")]),
query=multidict.MultiDict([("foo", "bar")]),
)
assert f[0] == "Query"
@ -175,7 +175,7 @@ Larry
def test_view_query(self):
d = ""
v = cv.ViewQuery()
f = v(d, query=ODict([("foo", "bar")]))
f = v(d, query=multidict.MultiDict([("foo", "bar")]))
assert f[0] == "Query"
assert [x for x in f[1]] == [[("header", "foo: "), ("text", "bar")]]

4
test/netlib/test_multidict.py
View File

@ -3,10 +3,6 @@ from netlib.multidict import MultiDict, ImmutableMultiDict, MultiDictView
class _TMulti(object):
@staticmethod
def _reduce_values(values):
return values[0]
@staticmethod
def _kconv(key):
return key.lower()

143
test/netlib/test_odict.py
View File

@ -1,143 +0,0 @@
from netlib import odict, tutils
class TestODict(object):
def test_repr(self):
h = odict.ODict()
h["one"] = ["two"]
assert repr(h)
def test_str_err(self):
h = odict.ODict()
with tutils.raises(ValueError):
h["key"] = u"foo"
with tutils.raises(ValueError):
h["key"] = b"foo"
def test_getset_state(self):
od = odict.ODict()
od.add("foo", 1)
od.add("foo", 2)
od.add("bar", 3)
state = od.get_state()
nd = odict.ODict.from_state(state)
assert nd == od
b = odict.ODict()
b.set_state(state)
assert b == od
def test_iter(self):
od = odict.ODict()
assert not [i for i in od]
od.add("foo", 1)
assert [i for i in od]
def test_keys(self):
od = odict.ODict()
assert not od.keys()
od.add("foo", 1)
assert od.keys() == ["foo"]
od.add("foo", 2)
assert od.keys() == ["foo"]
od.add("bar", 2)
assert len(od.keys()) == 2
def test_copy(self):
od = odict.ODict()
od.add("foo", 1)
od.add("foo", 2)
od.add("bar", 3)
assert od == od.copy()
assert not od != od.copy()
def test_del(self):
od = odict.ODict()
od.add("foo", 1)
od.add("Foo", 2)
od.add("bar", 3)
del od["foo"]
assert len(od.lst) == 2
def test_replace(self):
od = odict.ODict()
od.add("one", "two")
od.add("two", "one")
assert od.replace("one", "vun") == 2
assert od.lst == [
["vun", "two"],
["two", "vun"],
]
def test_get(self):
od = odict.ODict()
od.add("one", "two")
assert od.get("one") == ["two"]
assert od.get("two") is None
def test_get_first(self):
od = odict.ODict()
od.add("one", "two")
od.add("one", "three")
assert od.get_first("one") == "two"
assert od.get_first("two") is None
def test_extend(self):
a = odict.ODict([["a", "b"], ["c", "d"]])
b = odict.ODict([["a", "b"], ["e", "f"]])
a.extend(b)
assert len(a) == 4
assert a["a"] == ["b", "b"]
class TestODictCaseless(object):
def test_override(self):
o = odict.ODictCaseless()
o.add('T', 'application/x-www-form-urlencoded; charset=UTF-8')
o["T"] = ["foo"]
assert o["T"] == ["foo"]
def test_case_preservation(self):
od = odict.ODictCaseless()
od["Foo"] = ["1"]
assert "foo" in od
assert od.items()[0][0] == "Foo"
assert od.get("foo") == ["1"]
assert od.get("foo", [""]) == ["1"]
assert od.get("Foo", [""]) == ["1"]
assert od.get("xx", "yy") == "yy"
def test_del(self):
od = odict.ODictCaseless()
od.add("foo", 1)
od.add("Foo", 2)
od.add("bar", 3)
del od["foo"]
assert len(od) == 1
def test_keys(self):
od = odict.ODictCaseless()
assert not od.keys()
od.add("foo", 1)
assert od.keys() == ["foo"]
od.add("Foo", 2)
assert od.keys() == ["foo"]
od.add("bar", 2)
assert len(od.keys()) == 2
def test_add_order(self):
od = odict.ODict(
[
["one", "uno"],
["two", "due"],
["three", "tre"],
]
)
od["two"] = ["foo", "bar"]
assert od.lst == [
["one", "uno"],
["two", "foo"],
["three", "tre"],
["two", "bar"],
]