Skip to content

Commit

Permalink
Merge pull request #446 from wakabayashik/master
Browse files Browse the repository at this point in the history 
Improve rate control for download/upload speed limit
  • Loading branch information
@robinrodricks
robinrodricks authored Sep 9, 2019
2 parents cd96e11 + f0fd156 commit 108c637
Show file tree
Hide file tree
Showing 3 changed files with 98 additions and 38 deletions.
66 changes: 48 additions & 18 deletions FluentFTP/Client/FtpClient_HighLevelDownload.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -620,18 +620,32 @@ private bool DownloadFileInternal(string remotePath, Stream outStream, long rest
// we read until EOF instead of reading a specific number of bytes
var readToEnd = fileLen <= 0;

const int rateControlResolution = 100;
var rateLimitBytes = DownloadRateLimit != 0 ? (long)DownloadRateLimit * 1024 : 0;
var chunkSize = TransferChunkSize;
if (m_transferChunkSize == null && rateLimitBytes > 0) {
// reduce chunk size to optimize rate control
const int chunkSizeMin = 64;
while (chunkSize > chunkSizeMin) {
var chunkLenInMs = 1000L * chunkSize / rateLimitBytes;
if (chunkLenInMs <= rateControlResolution) {
break;
}
chunkSize = Math.Max(chunkSize >> 1, chunkSizeMin);
}
}

// loop till entire file downloaded
var buffer = new byte[TransferChunkSize];
var buffer = new byte[chunkSize];
var offset = restartPosition;

var transferStarted = DateTime.Now;
var sw = new Stopwatch();
long rateLimitBytes = DownloadRateLimit != 0 ? DownloadRateLimit * 1024 : 0;
while (offset < fileLen || readToEnd) {
try {
// read a chunk of bytes from the FTP stream
var readBytes = 1;
double limitCheckBytes = 0;
long limitCheckBytes = 0;
long bytesProcessed = 0;

sw.Start();
Expand All @@ -648,19 +662,20 @@ private bool DownloadFileInternal(string remotePath, Stream outStream, long rest
}

// honor the rate limit
var swTime = (int) sw.ElapsedMilliseconds;
if (rateLimitBytes > 0 && swTime >= 1000) {
var timeShouldTake = limitCheckBytes / rateLimitBytes * 1000;
var swTime = sw.ElapsedMilliseconds;
if (rateLimitBytes > 0) {
var timeShouldTake = limitCheckBytes * 1000 / rateLimitBytes;
if (timeShouldTake > swTime) {
#if CORE14
Task.Delay((int) (timeShouldTake - swTime)).Wait();
#else
Thread.Sleep((int) (timeShouldTake - swTime));
#endif
}

limitCheckBytes = 0;
sw.Restart();
else if (swTime > timeShouldTake + rateControlResolution) {
limitCheckBytes = 0;
sw.Restart();
}
}
}

Expand Down Expand Up @@ -738,19 +753,33 @@ private bool DownloadFileInternal(string remotePath, Stream outStream, long rest
// we read until EOF instead of reading a specific number of bytes
var readToEnd = fileLen <= 0;

const int rateControlResolution = 100;
var rateLimitBytes = DownloadRateLimit != 0 ? (long)DownloadRateLimit * 1024 : 0;
var chunkSize = TransferChunkSize;
if (m_transferChunkSize == null && rateLimitBytes > 0) {
// reduce chunk size to optimize rate control
const int chunkSizeMin = 64;
while (chunkSize > chunkSizeMin) {
var chunkLenInMs = 1000L * chunkSize / rateLimitBytes;
if (chunkLenInMs <= rateControlResolution) {
break;
}
chunkSize = Math.Max(chunkSize >> 1, chunkSizeMin);
}
}

// loop till entire file downloaded
var buffer = new byte[TransferChunkSize];
var buffer = new byte[chunkSize];
var offset = restartPosition;

var transferStarted = DateTime.Now;
var sw = new Stopwatch();
long rateLimitBytes = DownloadRateLimit != 0 ? DownloadRateLimit * 1024 : 0;

while (offset < fileLen || readToEnd) {
try {
// read a chunk of bytes from the FTP stream
var readBytes = 1;
double limitCheckBytes = 0;
long limitCheckBytes = 0;
long bytesProcessed = 0;

sw.Start();
Expand All @@ -767,16 +796,17 @@ private bool DownloadFileInternal(string remotePath, Stream outStream, long rest
}

// honor the rate limit
var swTime = (int) sw.ElapsedMilliseconds;
if (rateLimitBytes > 0 && swTime >= 1000) {
var timeShouldTake = limitCheckBytes / rateLimitBytes * 1000;
var swTime = sw.ElapsedMilliseconds;
if (rateLimitBytes > 0) {
var timeShouldTake = limitCheckBytes * 1000 / rateLimitBytes;
if (timeShouldTake > swTime) {
await Task.Delay((int) (timeShouldTake - swTime), token);
token.ThrowIfCancellationRequested();
}

limitCheckBytes = 0;
sw.Restart();
else if (swTime > timeShouldTake + rateControlResolution) {
limitCheckBytes = 0;
sw.Restart();
}
}
}

Expand Down
66 changes: 48 additions & 18 deletions FluentFTP/Client/FtpClient_HighLevelUpload.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -683,13 +683,27 @@ private bool UploadFileInternal(Stream fileData, string remotePath, bool createR
upStream = OpenAppend(remotePath, UploadDataType, checkFileExistsAgain);
}

const int rateControlResolution = 100;
var rateLimitBytes = UploadRateLimit != 0 ? (long)UploadRateLimit * 1024 : 0;
var chunkSize = TransferChunkSize;
if (m_transferChunkSize == null && rateLimitBytes > 0) {
// reduce chunk size to optimize rate control
const int chunkSizeMin = 64;
while (chunkSize > chunkSizeMin) {
var chunkLenInMs = 1000L * chunkSize / rateLimitBytes;
if (chunkLenInMs <= rateControlResolution) {
break;
}
chunkSize = Math.Max(chunkSize >> 1, chunkSizeMin);
}
}

// loop till entire file uploaded
var fileLen = fileData.Length;
var buffer = new byte[TransferChunkSize];
var buffer = new byte[chunkSize];

var transferStarted = DateTime.Now;
var sw = new Stopwatch();
long rateLimitBytes = UploadRateLimit != 0 ? UploadRateLimit * 1024 : 0;

// Fix #288 - Upload hangs with only a few bytes left
if (fileLen < upStream.Length) {
Expand All @@ -700,7 +714,7 @@ private bool UploadFileInternal(Stream fileData, string remotePath, bool createR
try {
// read a chunk of bytes from the file
int readBytes;
double limitCheckBytes = 0;
long limitCheckBytes = 0;
long bytesProcessed = 0;

sw.Start();
Expand All @@ -718,19 +732,20 @@ private bool UploadFileInternal(Stream fileData, string remotePath, bool createR
}

// honor the speed limit
var swTime = (int) sw.ElapsedMilliseconds;
if (rateLimitBytes > 0 && swTime >= 1000) {
var timeShouldTake = limitCheckBytes / rateLimitBytes * 1000;
var swTime = sw.ElapsedMilliseconds;
if (rateLimitBytes > 0) {
var timeShouldTake = limitCheckBytes * 1000 / rateLimitBytes;
if (timeShouldTake > swTime) {
#if CORE14
Task.Delay((int) (timeShouldTake - swTime)).Wait();
#else
Thread.Sleep((int) (timeShouldTake - swTime));
#endif
}

limitCheckBytes = 0;
sw.Restart();
else if (swTime > timeShouldTake + rateControlResolution) {
limitCheckBytes = 0;
sw.Restart();
}
}
}

Expand Down Expand Up @@ -877,13 +892,27 @@ private bool UploadFileInternal(Stream fileData, string remotePath, bool createR
upStream = await OpenAppendAsync(remotePath, UploadDataType, checkFileExistsAgain, token);
}

const int rateControlResolution = 100;
var rateLimitBytes = UploadRateLimit != 0 ? (long)UploadRateLimit * 1024 : 0;
var chunkSize = TransferChunkSize;
if (m_transferChunkSize == null && rateLimitBytes > 0) {
// reduce chunk size to optimize rate control
const int chunkSizeMin = 64;
while (chunkSize > chunkSizeMin) {
var chunkLenInMs = 1000L * chunkSize / rateLimitBytes;
if (chunkLenInMs <= rateControlResolution) {
break;
}
chunkSize = Math.Max(chunkSize >> 1, chunkSizeMin);
}
}

// loop till entire file uploaded
var fileLen = fileData.Length;
var buffer = new byte[TransferChunkSize];
var buffer = new byte[chunkSize];

var transferStarted = DateTime.Now;
var sw = new Stopwatch();
long rateLimitBytes = UploadRateLimit != 0 ? UploadRateLimit * 1024 : 0;

// Fix #288 - Upload hangs with only a few bytes left
if (fileLen < upStream.Length) {
Expand All @@ -894,7 +923,7 @@ private bool UploadFileInternal(Stream fileData, string remotePath, bool createR
try {
// read a chunk of bytes from the file
int readBytes;
double limitCheckBytes = 0;
long limitCheckBytes = 0;
long bytesProcessed = 0;

sw.Start();
Expand All @@ -912,16 +941,17 @@ private bool UploadFileInternal(Stream fileData, string remotePath, bool createR
}

// honor the rate limit
var swTime = (int) sw.ElapsedMilliseconds;
if (rateLimitBytes > 0 && swTime >= 1000) {
var timeShouldTake = limitCheckBytes / rateLimitBytes * 1000;
var swTime = sw.ElapsedMilliseconds;
if (rateLimitBytes > 0) {
var timeShouldTake = limitCheckBytes * 1000 / rateLimitBytes;
if (timeShouldTake > swTime) {
await Task.Delay((int) (timeShouldTake - swTime), token);
token.ThrowIfCancellationRequested();
}

limitCheckBytes = 0;
sw.Restart();
else if (swTime > timeShouldTake + rateControlResolution) {
limitCheckBytes = 0;
sw.Restart();
}
}
}

Expand Down
4 changes: 2 additions & 2 deletions FluentFTP/Client/FtpClient_Properties.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -644,15 +644,15 @@ public int BulkListingLength {
}


private int m_transferChunkSize = 65536;
private int? m_transferChunkSize;

/// <summary>
/// Gets or sets the number of bytes transferred in a single chunk (a single FTP command).
/// Used by <see cref="o:UploadFile"/>/<see cref="o:UploadFileAsync"/> and <see cref="o:DownloadFile"/>/<see cref="o:DownloadFileAsync"/>
/// to transfer large files in multiple chunks.
/// </summary>
public int TransferChunkSize {
get => m_transferChunkSize;
get => m_transferChunkSize ?? 65536;
set => m_transferChunkSize = value;
}

Expand Down

0 comments on commit 108c637

Please sign in to comment.