[System]: Refactor and cleanup WebResponseStream (#6662)

We are now using nested streams to take care of buffering, chunked encoding and automatic decompression.

GZip / Deflate decompression have been moved inside `WebResponseStream`, the `HttpWebResponse` now always returns a `WebResponseStream` (and not a `GZipStream`) because we have to do extra book-keeping after reading the entire content.
This commit is contained in:
Martin Baulig authored and Alexander Köplinger committed 2018-02-01 23:39:46 +01:00
1 parent 115915b3ca
commit 18a1a25345
13 files changed
+738 -516

No files matched your search

+1 -1
View File
@@ -1229,7 +1229,7 @@
<Compile Include="System.Net\ListenerAsyncResult.cs" />
<Compile Include="System.Net\ListenerPrefix.cs" />
<Compile Include="System.Net\MacProxy.cs" />
<Compile Include="System.Net\MonoChunkStream.cs" />
<Compile Include="System.Net\MonoChunkParser.cs" />
<Compile Include="System.Net\MonoHttpDate.cs" />
<Compile Include="System.Net\NetConfig.cs" />
<Compile Include="System.Net\NtlmClient.cs" />
+1 -1
View File
@@ -1229,7 +1229,7 @@
<Compile Include="System.Net\ListenerAsyncResult.cs" />
<Compile Include="System.Net\ListenerPrefix.cs" />
<Compile Include="System.Net\MacProxy.cs" />
<Compile Include="System.Net\MonoChunkStream.cs" />
<Compile Include="System.Net\MonoChunkParser.cs" />
<Compile Include="System.Net\MonoHttpDate.cs" />
<Compile Include="System.Net\NetConfig.cs" />
<Compile Include="System.Net\NtlmClient.cs" />
@@ -1229,7 +1229,7 @@
<Compile Include="System.Net\ListenerAsyncResult.cs" />
<Compile Include="System.Net\ListenerPrefix.cs" />
<Compile Include="System.Net\MacProxy.cs" />
<Compile Include="System.Net\MonoChunkStream.cs" />
<Compile Include="System.Net\MonoChunkParser.cs" />
<Compile Include="System.Net\MonoHttpDate.cs" />
<Compile Include="System.Net\NetConfig.cs" />
<Compile Include="System.Net\NtlmClient.cs" />
@@ -0,0 +1,77 @@
//
// BufferedReadStream.cs
//
// Author:
// Martin Baulig <[email protected]>
//
// Copyright (c) 2018 Xamarin Inc. (http://www.xamarin.com)
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
namespace System.Net
{
class BufferedReadStream : WebReadStream
{
readonly BufferOffsetSize readBuffer;
public BufferedReadStream (WebOperation operation, Stream innerStream,
BufferOffsetSize readBuffer)
: base (operation, innerStream)
{
this.readBuffer = readBuffer;
}
public override async Task<int> ReadAsync (byte[] buffer, int offset, int size,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested ();
if (buffer == null)
throw new ArgumentNullException (nameof (buffer));
int length = buffer.Length;
if (offset < 0 || length < offset)
throw new ArgumentOutOfRangeException (nameof (offset));
if (size < 0 || (length - offset) < size)
throw new ArgumentOutOfRangeException (nameof (size));
var remaining = readBuffer?.Size ?? 0;
if (remaining > 0) {
int copy = (remaining > size) ? size : remaining;
Buffer.BlockCopy (readBuffer.Buffer, readBuffer.Offset, buffer, offset, copy);
readBuffer.Offset += copy;
readBuffer.Size -= copy;
offset += copy;
size -= copy;
return copy;
}
if (InnerStream == null)
return 0;
return await InnerStream.ReadAsync (
buffer, offset, size, cancellationToken).ConfigureAwait (false);
}
}
}
@@ -31,7 +31,7 @@ using System.Runtime.InteropServices;
namespace System.Net {
class ChunkedInputStream : RequestStream {
bool disposed;
MonoChunkStream decoder;
MonoChunkParser decoder;
HttpListenerContext context;
bool no_more_data;
@@ -58,10 +58,10 @@ namespace System.Net {
{
this.context = context;
WebHeaderCollection coll = (WebHeaderCollection) context.Request.Headers;
decoder = new MonoChunkStream (coll);
decoder = new MonoChunkParser (coll);
}
public MonoChunkStream Decoder {
public MonoChunkParser Decoder {
get { return decoder; }
set { decoder = value; }
}
@@ -36,7 +36,6 @@ using System;
using System.Collections;
using System.Globalization;
using System.IO;
using System.IO.Compression;
using System.Net.Sockets;
using System.Runtime.Serialization;
using System.Threading;
@@ -98,16 +97,6 @@ namespace System.Net
this.cookie_container = container;
FillCookies ();
}
string content_encoding = webHeaders ["Content-Encoding"];
if (content_encoding == "gzip" && (stream.Request.AutomaticDecompression & DecompressionMethods.GZip) != 0) {
this.stream = new GZipStream (stream, CompressionMode.Decompress);
webHeaders.Remove (HttpRequestHeader.ContentEncoding);
}
else if (content_encoding == "deflate" && (stream.Request.AutomaticDecompression & DecompressionMethods.Deflate) != 0) {
this.stream = new DeflateStream (stream, CompressionMode.Decompress);
webHeaders.Remove (HttpRequestHeader.ContentEncoding);
}
}
[Obsolete ("Serialization is obsoleted for this type", false)]
@@ -0,0 +1,372 @@
//
// System.Net.MonoChunkParser
//
// Authors:
// Gonzalo Paniagua Javier ([email protected])
//
// (C) 2003 Ximian, Inc (http://www.ximian.com)
//
//
// Permission is hereby granted, free of charge, to any person obtaining
// a copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to
// permit persons to whom the Software is furnished to do so, subject to
// the following conditions:
//
// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
using System.Collections;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Text;
namespace System.Net
{
class MonoChunkParser
{
enum State {
None,
PartialSize,
Body,
BodyFinished,
Trailer
}
class Chunk {
public byte [] Bytes;
public int Offset;
public Chunk (byte [] chunk)
{
this.Bytes = chunk;
}
public int Read (byte [] buffer, int offset, int size)
{
int nread = (size > Bytes.Length - Offset) ? Bytes.Length - Offset : size;
Buffer.BlockCopy (Bytes, Offset, buffer, offset, nread);
Offset += nread;
return nread;
}
}
internal WebHeaderCollection headers;
int chunkSize;
int chunkRead;
int totalWritten;
State state;
//byte [] waitBuffer;
StringBuilder saved;
bool sawCR;
bool gotit;
int trailerState;
ArrayList chunks;
public MonoChunkParser (byte [] buffer, int offset, int size, WebHeaderCollection headers)
: this (headers)
{
Write (buffer, offset, size);
}
public MonoChunkParser (WebHeaderCollection headers)
{
this.headers = headers;
saved = new StringBuilder ();
chunks = new ArrayList ();
chunkSize = -1;
totalWritten = 0;
}
public void ResetBuffer ()
{
chunkSize = -1;
chunkRead = 0;
totalWritten = 0;
chunks.Clear ();
}
public void WriteAndReadBack (byte [] buffer, int offset, int size, ref int read)
{
if (offset + read > 0)
Write (buffer, offset, offset+read);
read = Read (buffer, offset, size);
}
public int Read (byte [] buffer, int offset, int size)
{
return ReadFromChunks (buffer, offset, size);
}
int ReadFromChunks (byte [] buffer, int offset, int size)
{
int count = chunks.Count;
int nread = 0;
var chunksForRemoving = new List<Chunk>(count);
for (int i = 0; i < count; i++) {
Chunk chunk = (Chunk) chunks [i];
if (chunk.Offset == chunk.Bytes.Length) {
chunksForRemoving.Add(chunk);
continue;
}
nread += chunk.Read (buffer, offset + nread, size - nread);
if (nread == size)
break;
}
foreach (var chunk in chunksForRemoving)
chunks.Remove(chunk);
return nread;
}
public void Write (byte [] buffer, int offset, int size)
{
if (offset < size)
InternalWrite (buffer, ref offset, size);
}
void InternalWrite (byte [] buffer, ref int offset, int size)
{
if (state == State.None || state == State.PartialSize) {
state = GetChunkSize (buffer, ref offset, size);
if (state == State.PartialSize)
return;
saved.Length = 0;
sawCR = false;
gotit = false;
}
if (state == State.Body && offset < size) {
state = ReadBody (buffer, ref offset, size);
if (state == State.Body)
return;
}
if (state == State.BodyFinished && offset < size) {
state = ReadCRLF (buffer, ref offset, size);
if (state == State.BodyFinished)
return;
sawCR = false;
}
if (state == State.Trailer && offset < size) {
state = ReadTrailer (buffer, ref offset, size);
if (state == State.Trailer)
return;
saved.Length = 0;
sawCR = false;
gotit = false;
}
if (offset < size)
InternalWrite (buffer, ref offset, size);
}
public bool WantMore {
get { return (chunkRead != chunkSize || chunkSize != 0 || state != State.None); }
}
public bool DataAvailable {
get {
int count = chunks.Count;
for (int i = 0; i < count; i++) {
Chunk ch = (Chunk) chunks [i];
if (ch == null || ch.Bytes == null)
continue;
if (ch.Bytes.Length > 0 && ch.Offset < ch.Bytes.Length)
return (state != State.Body);
}
return false;
}
}
public int TotalDataSize {
get { return totalWritten; }
}
public int ChunkLeft {
get { return chunkSize - chunkRead; }
}
State ReadBody (byte [] buffer, ref int offset, int size)
{
if (chunkSize == 0)
return State.BodyFinished;
int diff = size - offset;
if (diff + chunkRead > chunkSize)
diff = chunkSize - chunkRead;
byte [] chunk = new byte [diff];
Buffer.BlockCopy (buffer, offset, chunk, 0, diff);
chunks.Add (new Chunk (chunk));
offset += diff;
chunkRead += diff;
totalWritten += diff;
return (chunkRead == chunkSize) ? State.BodyFinished : State.Body;
}
State GetChunkSize (byte [] buffer, ref int offset, int size)
{
chunkRead = 0;
chunkSize = 0;
char c = '\0';
while (offset < size) {
c = (char) buffer [offset++];
if (c == '\r') {
if (sawCR)
ThrowProtocolViolation ("2 CR found");
sawCR = true;
continue;
}
if (sawCR && c == '\n')
break;
if (c == ' ')
gotit = true;
if (!gotit)
saved.Append (c);
if (saved.Length > 20)
ThrowProtocolViolation ("chunk size too long.");
}
if (!sawCR || c != '\n') {
if (offset < size)
ThrowProtocolViolation ("Missing \\n");
try {
if (saved.Length > 0) {
chunkSize = Int32.Parse (RemoveChunkExtension (saved.ToString ()), NumberStyles.HexNumber);
}
} catch (Exception) {
ThrowProtocolViolation ("Cannot parse chunk size.");
}
return State.PartialSize;
}
chunkRead = 0;
try {
chunkSize = Int32.Parse (RemoveChunkExtension (saved.ToString ()), NumberStyles.HexNumber);
} catch (Exception) {
ThrowProtocolViolation ("Cannot parse chunk size.");
}
if (chunkSize == 0) {
trailerState = 2;
return State.Trailer;
}
return State.Body;
}
static string RemoveChunkExtension (string input)
{
int idx = input.IndexOf (';');
if (idx == -1)
return input;
return input.Substring (0, idx);
}
State ReadCRLF (byte [] buffer, ref int offset, int size)
{
if (!sawCR) {
if ((char) buffer [offset++] != '\r')
ThrowProtocolViolation ("Expecting \\r");
sawCR = true;
if (offset == size)
return State.BodyFinished;
}
if (sawCR && (char) buffer [offset++] != '\n')
ThrowProtocolViolation ("Expecting \\n");
return State.None;
}
State ReadTrailer (byte [] buffer, ref int offset, int size)
{
char c = '\0';
// short path
if (trailerState == 2 && (char) buffer [offset] == '\r' && saved.Length == 0) {
offset++;
if (offset < size && (char) buffer [offset] == '\n') {
offset++;
return State.None;
}
offset--;
}
int st = trailerState;
string stString = "\r\n\r";
while (offset < size && st < 4) {
c = (char) buffer [offset++];
if ((st == 0 || st == 2) && c == '\r') {
st++;
continue;
}
if ((st == 1 || st == 3) && c == '\n') {
st++;
continue;
}
if (st > 0) {
saved.Append (stString.Substring (0, saved.Length == 0? st-2: st));
st = 0;
if (saved.Length > 4196)
ThrowProtocolViolation ("Error reading trailer (too long).");
}
}
if (st < 4) {
trailerState = st;
if (offset < size)
ThrowProtocolViolation ("Error reading trailer.");
return State.Trailer;
}
StringReader reader = new StringReader (saved.ToString ());
string line;
while ((line = reader.ReadLine ()) != null && line != "")
headers.Add (line);
return State.None;
}
static void ThrowProtocolViolation (string message)
{
WebException we = new WebException (message, null, WebExceptionStatus.ServerProtocolViolation, null);
throw we;
}
}
}
+65 -342
View File
@@ -1,371 +1,94 @@
//
// System.Net.MonoChunkStream
// MonoChunkStream.cs
//
// Authors:
// Gonzalo Paniagua Javier (gonzalo@ximian.com)
// Author:
// Martin Baulig <mabaul@microsoft.com>
//
// (C) 2003 Ximian, Inc (http://www.ximian.com)
// Copyright (c) 2018 Xamarin Inc. (http://www.xamarin.com)
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// Permission is hereby granted, free of charge, to any person obtaining
// a copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to
// permit persons to whom the Software is furnished to do so, subject to
// the following conditions:
//
// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
using System.Collections;
using System.Collections.Generic;
using System.Globalization;
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
using System;
using System.IO;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace System.Net
{
class MonoChunkStream
class MonoChunkStream : WebReadStream
{
enum State {
None,
PartialSize,
Body,
BodyFinished,
Trailer
protected WebHeaderCollection Headers {
get;
}
class Chunk {
public byte [] Bytes;
public int Offset;
public Chunk (byte [] chunk)
{
this.Bytes = chunk;
}
public int Read (byte [] buffer, int offset, int size)
{
int nread = (size > Bytes.Length - Offset) ? Bytes.Length - Offset : size;
Buffer.BlockCopy (Bytes, Offset, buffer, offset, nread);
Offset += nread;
return nread;
}
protected MonoChunkParser Decoder {
get;
}
internal WebHeaderCollection headers;
int chunkSize;
int chunkRead;
int totalWritten;
State state;
//byte [] waitBuffer;
StringBuilder saved;
bool sawCR;
bool gotit;
int trailerState;
ArrayList chunks;
public MonoChunkStream (byte [] buffer, int offset, int size, WebHeaderCollection headers)
: this (headers)
public MonoChunkStream (WebOperation operation, Stream innerStream,
WebHeaderCollection headers)
: base (operation, innerStream)
{
Write (buffer, offset, size);
Headers = headers;
Decoder = new MonoChunkParser (headers);
}
public MonoChunkStream (WebHeaderCollection headers)
public override async Task<int> ReadAsync (byte[] buffer, int offset, int size,
CancellationToken cancellationToken)
{
this.headers = headers;
saved = new StringBuilder ();
chunks = new ArrayList ();
chunkSize = -1;
totalWritten = 0;
}
cancellationToken.ThrowIfCancellationRequested ();
public void ResetBuffer ()
{
chunkSize = -1;
chunkRead = 0;
totalWritten = 0;
chunks.Clear ();
}
public void WriteAndReadBack (byte [] buffer, int offset, int size, ref int read)
{
if (offset + read > 0)
Write (buffer, offset, offset+read);
read = Read (buffer, offset, size);
}
if (buffer == null)
throw new ArgumentNullException (nameof (buffer));
public int Read (byte [] buffer, int offset, int size)
{
return ReadFromChunks (buffer, offset, size);
}
int length = buffer.Length;
if (offset < 0 || length < offset)
throw new ArgumentOutOfRangeException (nameof (offset));
if (size < 0 || (length - offset) < size)
throw new ArgumentOutOfRangeException (nameof (size));
int ReadFromChunks (byte [] buffer, int offset, int size)
{
int count = chunks.Count;
int nread = 0;
if (Decoder.DataAvailable)
return Decoder.Read (buffer, offset, size);
var chunksForRemoving = new List<Chunk>(count);
for (int i = 0; i < count; i++) {
Chunk chunk = (Chunk) chunks [i];
int ret = 0;
byte[] moreBytes = null;
while (ret == 0 && Decoder.WantMore) {
int localSize = Decoder.ChunkLeft;
if (localSize <= 0) // not read chunk size yet
localSize = 1024;
else if (localSize > 16384)
localSize = 16384;
if (chunk.Offset == chunk.Bytes.Length) {
chunksForRemoving.Add(chunk);
continue;
}
nread += chunk.Read (buffer, offset + nread, size - nread);
if (nread == size)
break;
if (moreBytes == null || moreBytes.Length < localSize)
moreBytes = new byte[localSize];
ret = await InnerStream.ReadAsync (
moreBytes, 0, localSize, cancellationToken).ConfigureAwait (false);
if (ret <= 0)
return ret;
Decoder.Write (moreBytes, 0, ret);
ret = Decoder.Read (buffer, offset, size);
}
foreach (var chunk in chunksForRemoving)
chunks.Remove(chunk);
return nread;
}
public void Write (byte [] buffer, int offset, int size)
{
if (offset < size)
InternalWrite (buffer, ref offset, size);
}
void InternalWrite (byte [] buffer, ref int offset, int size)
{
if (state == State.None || state == State.PartialSize) {
state = GetChunkSize (buffer, ref offset, size);
if (state == State.PartialSize)
return;
saved.Length = 0;
sawCR = false;
gotit = false;
}
if (state == State.Body && offset < size) {
state = ReadBody (buffer, ref offset, size);
if (state == State.Body)
return;
}
if (state == State.BodyFinished && offset < size) {
state = ReadCRLF (buffer, ref offset, size);
if (state == State.BodyFinished)
return;
sawCR = false;
}
if (state == State.Trailer && offset < size) {
state = ReadTrailer (buffer, ref offset, size);
if (state == State.Trailer)
return;
saved.Length = 0;
sawCR = false;
gotit = false;
}
if (offset < size)
InternalWrite (buffer, ref offset, size);
}
public bool WantMore {
get { return (chunkRead != chunkSize || chunkSize != 0 || state != State.None); }
}
public bool DataAvailable {
get {
int count = chunks.Count;
for (int i = 0; i < count; i++) {
Chunk ch = (Chunk) chunks [i];
if (ch == null || ch.Bytes == null)
continue;
if (ch.Bytes.Length > 0 && ch.Offset < ch.Bytes.Length)
return (state != State.Body);
}
return false;
}
}
public int TotalDataSize {
get { return totalWritten; }
}
public int ChunkLeft {
get { return chunkSize - chunkRead; }
}
State ReadBody (byte [] buffer, ref int offset, int size)
{
if (chunkSize == 0)
return State.BodyFinished;
int diff = size - offset;
if (diff + chunkRead > chunkSize)
diff = chunkSize - chunkRead;
byte [] chunk = new byte [diff];
Buffer.BlockCopy (buffer, offset, chunk, 0, diff);
chunks.Add (new Chunk (chunk));
offset += diff;
chunkRead += diff;
totalWritten += diff;
return (chunkRead == chunkSize) ? State.BodyFinished : State.Body;
}
State GetChunkSize (byte [] buffer, ref int offset, int size)
{
chunkRead = 0;
chunkSize = 0;
char c = '\0';
while (offset < size) {
c = (char) buffer [offset++];
if (c == '\r') {
if (sawCR)
ThrowProtocolViolation ("2 CR found");
sawCR = true;
continue;
}
if (sawCR && c == '\n')
break;
if (c == ' ')
gotit = true;
if (!gotit)
saved.Append (c);
if (saved.Length > 20)
ThrowProtocolViolation ("chunk size too long.");
}
if (!sawCR || c != '\n') {
if (offset < size)
ThrowProtocolViolation ("Missing \\n");
try {
if (saved.Length > 0) {
chunkSize = Int32.Parse (RemoveChunkExtension (saved.ToString ()), NumberStyles.HexNumber);
}
} catch (Exception) {
ThrowProtocolViolation ("Cannot parse chunk size.");
}
return State.PartialSize;
}
chunkRead = 0;
try {
chunkSize = Int32.Parse (RemoveChunkExtension (saved.ToString ()), NumberStyles.HexNumber);
} catch (Exception) {
ThrowProtocolViolation ("Cannot parse chunk size.");
}
if (chunkSize == 0) {
trailerState = 2;
return State.Trailer;
}
return State.Body;
}
static string RemoveChunkExtension (string input)
{
int idx = input.IndexOf (';');
if (idx == -1)
return input;
return input.Substring (0, idx);
}
State ReadCRLF (byte [] buffer, ref int offset, int size)
{
if (!sawCR) {
if ((char) buffer [offset++] != '\r')
ThrowProtocolViolation ("Expecting \\r");
sawCR = true;
if (offset == size)
return State.BodyFinished;
}
if (sawCR && (char) buffer [offset++] != '\n')
ThrowProtocolViolation ("Expecting \\n");
return State.None;
}
State ReadTrailer (byte [] buffer, ref int offset, int size)
{
char c = '\0';
// short path
if (trailerState == 2 && (char) buffer [offset] == '\r' && saved.Length == 0) {
offset++;
if (offset < size && (char) buffer [offset] == '\n') {
offset++;
return State.None;
}
offset--;
}
int st = trailerState;
string stString = "\r\n\r";
while (offset < size && st < 4) {
c = (char) buffer [offset++];
if ((st == 0 || st == 2) && c == '\r') {
st++;
continue;
}
if ((st == 1 || st == 3) && c == '\n') {
st++;
continue;
}
if (st > 0) {
saved.Append (stString.Substring (0, saved.Length == 0? st-2: st));
st = 0;
if (saved.Length > 4196)
ThrowProtocolViolation ("Error reading trailer (too long).");
}
}
if (st < 4) {
trailerState = st;
if (offset < size)
ThrowProtocolViolation ("Error reading trailer.");
return State.Trailer;
}
StringReader reader = new StringReader (saved.ToString ());
string line;
while ((line = reader.ReadLine ()) != null && line != "")
headers.Add (line);
return State.None;
}
static void ThrowProtocolViolation (string message)
{
WebException we = new WebException (message, null, WebExceptionStatus.ServerProtocolViolation, null);
throw we;
return ret;
}
}
}
@@ -242,7 +242,7 @@ namespace System.Net
repeat = SchedulerIteration (defaultGroup);
Debug ($"ITERATION #1: {repeat} {groups != null}");
Debug ($"ITERATION #1: repeat={repeat} groups={groups?.Count}");
if (groups != null) {
foreach (var group in groups) {
@@ -251,7 +251,7 @@ namespace System.Net
}
}
Debug ($"ITERATION #3: {repeat}");
Debug ($"ITERATION #3: repeat={repeat}");
} while (repeat);
}
@@ -324,9 +324,10 @@ namespace System.Net
// Is there anything in the queue?
var next = group.GetNextOperation ();
Debug ($"{me} no pending operations.");
if (next == null)
if (next == null) {
Debug ($"{me} no pending operations.");
return false;
}
Debug ($"{me} found pending operation Op={next.ID}");
@@ -0,0 +1,149 @@
//
// WebReadStream.cs
//
// Author:
// Martin Baulig <[email protected]>
//
// Copyright (c) 2018 Xamarin Inc. (http://www.xamarin.com)
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
namespace System.Net
{
abstract class WebReadStream : Stream
{
public WebOperation Operation {
get;
}
protected Stream InnerStream {
get;
}
public WebReadStream (WebOperation operation, Stream innerStream)
{
Operation = operation;
InnerStream = innerStream;
}
Exception NotSupported => new NotSupportedException ();
public override long Length => throw NotSupported;
public override long Position {
get => throw NotSupported;
set => throw NotSupported;
}
public override bool CanSeek => false;
public override bool CanRead => true;
public override bool CanWrite => false;
public override void SetLength (long value)
{
throw NotSupported;
}
public override long Seek (long offset, SeekOrigin origin)
{
throw NotSupported;
}
public override void Write (byte[] buffer, int offset, int count)
{
throw NotSupported;
}
public override void Flush ()
{
throw NotSupported;
}
protected Exception GetException (Exception e)
{
e = HttpWebRequest.FlattenException (e);
if (e is WebException)
return e;
if (Operation.Aborted || e is OperationCanceledException || e is ObjectDisposedException)
return HttpWebRequest.CreateRequestAbortedException ();
return e;
}
public override int Read (byte[] buffer, int offset, int size)
{
if (!CanRead)
throw new NotSupportedException (SR.net_writeonlystream);
Operation.ThrowIfClosedOrDisposed ();
if (buffer == null)
throw new ArgumentNullException (nameof (buffer));
int length = buffer.Length;
if (offset < 0 || length < offset)
throw new ArgumentOutOfRangeException (nameof (offset));
if (size < 0 || (length - offset) < size)
throw new ArgumentOutOfRangeException (nameof (size));
try {
return ReadAsync (buffer, offset, size, CancellationToken.None).Result;
} catch (Exception e) {
throw GetException (e);
}
}
public override IAsyncResult BeginRead (byte[] buffer, int offset, int size,
AsyncCallback cb, object state)
{
if (!CanRead)
throw new NotSupportedException (SR.net_writeonlystream);
Operation.ThrowIfClosedOrDisposed ();
if (buffer == null)
throw new ArgumentNullException (nameof (buffer));
int length = buffer.Length;
if (offset < 0 || length < offset)
throw new ArgumentOutOfRangeException (nameof (offset));
if (size < 0 || (length - offset) < size)
throw new ArgumentOutOfRangeException (nameof (size));
var task = ReadAsync (buffer, offset, size, CancellationToken.None);
return TaskToApm.Begin (task, cb, state);
}
public override int EndRead (IAsyncResult r)
{
if (r == null)
throw new ArgumentNullException (nameof (r));
try {
return TaskToApm.End<int> (r);
} catch (Exception e) {
throw GetException (e);
}
}
}
}
+60 -152
View File
@@ -24,6 +24,7 @@
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
using System.IO;
using System.IO.Compression;
using System.Text;
using System.Collections;
using System.Collections.Generic;
@@ -36,7 +37,7 @@ namespace System.Net
{
class WebResponseStream : WebConnectionStream
{
BufferOffsetSize readBuffer;
Stream innerStreamWrapper;
long contentLength;
long totalRead;
bool nextReadCalled;
@@ -98,14 +99,8 @@ namespace System.Net
public override bool CanWrite => false;
protected bool ChunkedRead {
get;
private set;
}
protected MonoChunkStream ChunkStream {
get;
private set;
bool ChunkedRead {
get; set;
}
public override async Task<int> ReadAsync (byte[] buffer, int offset, int size, CancellationToken cancellationToken)
@@ -140,12 +135,12 @@ namespace System.Net
WebConnection.Debug ($"{ME} READ ASYNC #2: {totalRead} {contentLength}");
int oldBytes = 0, nbytes = 0;
int nbytes = 0;
Exception throwMe = null;
try {
// FIXME: NetworkStream.ReadAsync() does not support cancellation.
(oldBytes, nbytes) = await HttpWebRequest.RunWithTimeout (
nbytes = await HttpWebRequest.RunWithTimeout (
ct => ProcessRead (buffer, offset, size, ct),
ReadTimeout, () => {
Operation.Abort ();
@@ -155,7 +150,7 @@ namespace System.Net
throwMe = GetReadException (WebExceptionStatus.ReceiveFailure, e, "ReadAsync");
}
WebConnection.Debug ($"{ME} READ ASYNC #3: {totalRead} {contentLength} - {oldBytes} {nbytes} {throwMe?.Message}");
WebConnection.Debug ($"{ME} READ ASYNC #3: {totalRead} {contentLength} - {nbytes} {throwMe?.Message}");
if (throwMe != null) {
lock (locker) {
@@ -170,133 +165,65 @@ namespace System.Net
}
lock (locker) {
readTcs.TrySetResult (oldBytes + nbytes);
readTcs.TrySetResult (nbytes);
readTcs = null;
nestedRead = 0;
}
if (totalRead >= contentLength && !nextReadCalled) {
WebConnection.Debug ($"{ME} READ ASYNC - READ COMPLETE: {oldBytes} {nbytes} - {totalRead} {contentLength} {nextReadCalled}");
WebConnection.Debug ($"{ME} READ ASYNC - READ COMPLETE: {nbytes} - {totalRead} {contentLength} {nextReadCalled}");
if (!nextReadCalled) {
nextReadCalled = true;
Operation.CompleteResponseRead (true);
}
}
return oldBytes + nbytes;
return nbytes;
}
async Task<(int, int)> ProcessRead (byte[] buffer, int offset, int size, CancellationToken cancellationToken)
async Task<int> ProcessRead (byte[] buffer, int offset, int size, CancellationToken cancellationToken)
{
WebConnection.Debug ($"{ME} PROCESS READ: {totalRead} {contentLength}");
cancellationToken.ThrowIfCancellationRequested ();
if (totalRead >= contentLength) {
if (read_eof || totalRead >= contentLength) {
read_eof = true;
contentLength = totalRead;
return (0, 0);
}
int oldBytes = 0;
int remaining = readBuffer?.Size ?? 0;
if (remaining > 0) {
int copy = (remaining > size) ? size : remaining;
Buffer.BlockCopy (readBuffer.Buffer, readBuffer.Offset, buffer, offset, copy);
readBuffer.Offset += copy;
readBuffer.Size -= copy;
offset += copy;
size -= copy;
totalRead += copy;
if (totalRead >= contentLength) {
contentLength = totalRead;
read_eof = true;
}
if (size == 0 || totalRead >= contentLength)
return (0, copy);
oldBytes = copy;
return 0;
}
if (contentLength != Int64.MaxValue && contentLength - totalRead < size)
size = (int)(contentLength - totalRead);
WebConnection.Debug ($"{ME} PROCESS READ #1: {oldBytes} {size} {read_eof}");
if (read_eof) {
contentLength = totalRead;
return (oldBytes, 0);
}
WebConnection.Debug ($"{ME} PROCESS READ #1: {size} {read_eof}");
var ret = await InnerReadAsync (buffer, offset, size, cancellationToken).ConfigureAwait (false);
if (ret <= 0) {
read_eof = true;
contentLength = totalRead;
return (oldBytes, 0);
return ret;
}
totalRead += ret;
return (oldBytes, ret);
return ret;
}
internal async Task<int> InnerReadAsync (byte[] buffer, int offset, int size, CancellationToken cancellationToken)
async Task<int> InnerReadAsync (byte[] buffer, int offset, int size, CancellationToken cancellationToken)
{
WebConnection.Debug ($"{ME} INNER READ ASYNC");
Operation.ThrowIfDisposed (cancellationToken);
int nbytes = 0;
bool done = false;
if (!ChunkedRead || (!ChunkStream.DataAvailable && ChunkStream.WantMore)) {
nbytes = await InnerStream.ReadAsync (buffer, offset, size, cancellationToken).ConfigureAwait (false);
WebConnection.Debug ($"{ME} INNER READ ASYNC #1: {nbytes} {ChunkedRead}");
if (!ChunkedRead)
return nbytes;
done = nbytes == 0;
}
WebConnection.Debug ($"{ME} INNER READ ASYNC: stream={innerStreamWrapper}");
try {
ChunkStream.WriteAndReadBack (buffer, offset, size, ref nbytes);
WebConnection.Debug ($"{ME} INNER READ ASYNC #1: {done} {nbytes} {ChunkStream.WantMore}");
if (!done && nbytes == 0 && ChunkStream.WantMore)
nbytes = await EnsureReadAsync (buffer, offset, size, cancellationToken).ConfigureAwait (false);
} catch (Exception e) {
if (e is WebException || e is OperationCanceledException)
throw;
throw new WebException ("Invalid chunked data.", e, WebExceptionStatus.ServerProtocolViolation, null);
var ret = await innerStreamWrapper.ReadAsync (
buffer, offset, size, cancellationToken).ConfigureAwait (false);
WebConnection.Debug ($"{ME} INNER READ ASYNC DONE: {ret}");
return ret;
} catch (Exception ex) {
WebConnection.Debug ($"{ME} INNER READ ASYNC EX: {ex.Message}");
throw;
}
if ((done || nbytes == 0) && ChunkStream.ChunkLeft != 0) {
// HandleError (WebExceptionStatus.ReceiveFailure, null, "chunked EndRead");
throw new WebException ("Read error", null, WebExceptionStatus.ReceiveFailure, null);
}
return nbytes;
}
async Task<int> EnsureReadAsync (byte[] buffer, int offset, int size, CancellationToken cancellationToken)
{
byte[] morebytes = null;
int nbytes = 0;
while (nbytes == 0 && ChunkStream.WantMore && !cancellationToken.IsCancellationRequested) {
int localsize = ChunkStream.ChunkLeft;
if (localsize <= 0) // not read chunk size yet
localsize = 1024;
else if (localsize > 16384)
localsize = 16384;
if (morebytes == null || morebytes.Length < localsize)
morebytes = new byte[localsize];
int nread = await InnerStream.ReadAsync (morebytes, 0, localsize, cancellationToken).ConfigureAwait (false);
if (nread <= 0)
return 0; // Error
ChunkStream.Write (morebytes, 0, nread);
nbytes += ChunkStream.Read (buffer, offset + nbytes, size - nbytes);
}
return nbytes;
}
bool CheckAuthHeader (string headerName)
@@ -321,7 +248,7 @@ namespace System.Net
}
}
async Task Initialize (BufferOffsetSize buffer, CancellationToken cancellationToken)
void Initialize (BufferOffsetSize buffer)
{
WebConnection.Debug ($"{ME} INIT: status={(int)StatusCode} bos={buffer.Offset}/{buffer.Size}");
@@ -350,35 +277,28 @@ namespace System.Net
if (!Int32.TryParse (clength, out stream_length))
stream_length = -1;
string me = "WebResponseStream.Initialize()";
string tencoding = null;
if (ExpectContent)
tencoding = Headers["Transfer-Encoding"];
ChunkedRead = (tencoding != null && tencoding.IndexOf ("chunked", StringComparison.OrdinalIgnoreCase) != -1);
if (!ChunkedRead) {
readBuffer = buffer;
try {
if (contentLength > 0 && readBuffer.Size >= contentLength) {
if (!IsNtlmAuth ())
await ReadAllAsync (false, cancellationToken).ConfigureAwait (false);
}
} catch (Exception e) {
throw GetReadException (WebExceptionStatus.ReceiveFailure, e, me);
}
} else if (ChunkStream == null) {
try {
ChunkStream = new MonoChunkStream (buffer.Buffer, buffer.Offset, buffer.Offset + buffer.Size, Headers);
} catch (Exception e) {
throw GetReadException (WebExceptionStatus.ServerProtocolViolation, e, me);
}
if (ChunkedRead) {
innerStreamWrapper = new MonoChunkStream (
Operation, CreateStreamWrapper (buffer), Headers);
} else if (!IsNtlmAuth () && contentLength > 0 && buffer.Size >= contentLength) {
innerStreamWrapper = new BufferedReadStream (Operation, null, buffer);
} else {
ChunkStream.ResetBuffer ();
try {
ChunkStream.Write (buffer.Buffer, buffer.Offset, buffer.Size);
} catch (Exception e) {
throw GetReadException (WebExceptionStatus.ServerProtocolViolation, e, me);
}
innerStreamWrapper = CreateStreamWrapper (buffer);
}
string content_encoding = Headers["Content-Encoding"];
if (content_encoding == "gzip" && (Request.AutomaticDecompression & DecompressionMethods.GZip) != 0) {
innerStreamWrapper = new GZipStream (innerStreamWrapper, CompressionMode.Decompress);
Headers.Remove (HttpRequestHeader.ContentEncoding);
} else if (content_encoding == "deflate" && (Request.AutomaticDecompression & DecompressionMethods.Deflate) != 0) {
innerStreamWrapper = new DeflateStream (innerStreamWrapper, CompressionMode.Decompress);
Headers.Remove (HttpRequestHeader.ContentEncoding);
}
WebConnection.Debug ($"{ME} INIT #1: - {ExpectContent} {closed} {nextReadCalled}");
@@ -393,6 +313,13 @@ namespace System.Net
}
}
Stream CreateStreamWrapper (BufferOffsetSize buffer)
{
if (buffer == null || buffer.Size == 0)
return InnerStream;
return new BufferedReadStream (Operation, InnerStream, buffer);
}
internal async Task ReadAllAsync (bool resending, CancellationToken cancellationToken)
{
WebConnection.Debug ($"{ME} READ ALL ASYNC: resending={resending} eof={read_eof} total={totalRead} " +
@@ -430,7 +357,6 @@ namespace System.Net
if (totalRead >= contentLength)
return;
byte[] b = null;
int new_size;
if (contentLength == Int64.MaxValue && !ChunkedRead) {
@@ -451,19 +377,10 @@ namespace System.Net
KeepAlive = false;
}
BufferOffsetSize bos;
if (contentLength == Int64.MaxValue) {
MemoryStream ms = new MemoryStream ();
BufferOffsetSize buffer = null;
if (readBuffer != null && readBuffer.Size > 0) {
ms.Write (readBuffer.Buffer, readBuffer.Offset, readBuffer.Size);
readBuffer.Offset = 0;
readBuffer.Size = readBuffer.Buffer.Length;
if (readBuffer.Buffer.Length >= 8192)
buffer = readBuffer;
}
if (buffer == null)
buffer = new BufferOffsetSize (new byte[8192], false);
var ms = new MemoryStream ();
var buffer = new BufferOffsetSize (new byte[8192], false);
int read;
while ((read = await InnerReadAsync (buffer.Buffer, buffer.Offset, buffer.Size, cancellationToken)) != 0)
@@ -471,29 +388,23 @@ namespace System.Net
new_size = (int)ms.Length;
contentLength = new_size;
readBuffer = new BufferOffsetSize (ms.GetBuffer (), 0, new_size, false);
bos = new BufferOffsetSize (ms.GetBuffer (), 0, new_size, false);
} else {
new_size = (int)(contentLength - totalRead);
b = new byte[new_size];
var b = new byte[new_size];
int readSize = 0;
if (readBuffer != null && readBuffer.Size > 0) {
readSize = readBuffer.Size;
if (readSize > new_size)
readSize = new_size;
Buffer.BlockCopy (readBuffer.Buffer, readBuffer.Offset, b, 0, readSize);
}
int remaining = new_size - readSize;
int remaining = new_size;
int r = -1;
while (remaining > 0 && r != 0) {
r = await InnerReadAsync (b, readSize, remaining, cancellationToken);
r = await InnerReadAsync (b, readSize, remaining, cancellationToken).ConfigureAwait (false);
remaining -= r;
readSize += r;
}
bos = new BufferOffsetSize (b, 0, new_size, false);
}
readBuffer = new BufferOffsetSize (b, 0, new_size, false);
innerStreamWrapper = new BufferedReadStream (Operation, null, bos);
totalRead = 0;
nextReadCalled = true;
myReadTcs.TrySetResult (new_size);
@@ -605,8 +516,7 @@ namespace System.Net
WebConnection.Debug ($"{ME} INIT READ ASYNC LOOP DONE: {buffer.Offset} {buffer.Size}");
try {
Operation.ThrowIfDisposed (cancellationToken);
await Initialize (buffer, cancellationToken).ConfigureAwait (false);
Initialize (buffer);
} catch (Exception e) {
throw GetReadException (WebExceptionStatus.ReceiveFailure, e, "ReadDoneAsync6");
}
@@ -727,7 +637,5 @@ namespace System.Net
throw GetReadException (WebExceptionStatus.ServerProtocolViolation, null, "GetResponse");
}
}
}
+1 -1
View File
@@ -44,7 +44,7 @@ System.IO/WatcherChangeTypes.cs
System.Net/BasicClient.cs
System.Net/BindIPEndPoint.cs
System.Net/MonoChunkStream.cs
System.Net/MonoChunkParser.cs
System.Net/DecompressionMethods.cs
System.Net/DefaultCertificatePolicy.cs
System.Net/DigestClient.cs
@@ -49,6 +49,9 @@ System.Net/WebConnectionTunnel.cs
System.Net/WebOperation.cs
System.Net/WebRequestStream.cs
System.Net/WebResponseStream.cs
System.Net/MonoChunkStream.cs
System.Net/WebReadStream.cs
System.Net/BufferedReadStream.cs
../referencesource/System/net/System/Net/Sockets/TCPClient.cs
../referencesource/System/net/System/Net/Sockets/TCPListener.cs