简单实现TCP下的大文件高效传输
在TCP下进行大文件传输不象小文件那样直接打包个BUFFER发送出去,因为文件比较大所以不可能把文件读到一个BUFFER发送出去.主要有些文件的大小可能是1G,2G或更大,分配这么大的BUFFER对内存来说显然是不现实的事情;针对服务端的设计来说就更需要严紧些,BUFFER大小的限制也是变得很重要.下面介绍使用Beetle简单地实现大文件在TCP的传输应用.
协议制定
既然需要把文件分块来处理,那在TCP传输的过程需要制定一些协议来规范数据有效性,数据协议主要有三个:告诉服务器需要上传文件,文件块上传和返回每个环节处理的结果.
1)上传文件指令
public class Upload:ObjectMessage
{
public string FileMD5
{
get;
set;
}
public string FileName
{
get;
set;
}
public long FileSize
{
get;
set;
}
public override void FromProtocolData(HttpData httpbase)
{
FileName = httpbase[CONSTVALUE.HEADER_NAME];
FileMD5 = httpbase[CONSTVALUE.HEADER_MD5];
FileSize = long.Parse(httpbase[CONSTVALUE.HEADER_FILESIZE]);
}
protected override void OnDisposed()
{
}
protected override void OnToProtocolData(HttpData httpbase)
{
httpbase.Command = CONSTVALUE.COMMAND_UPLOAD;
httpbase[CONSTVALUE.HEADER_MD5] = FileMD5;
httpbase[CONSTVALUE.HEADER_NAME] = FileName;
httpbase[CONSTVALUE.HEADER_FILESIZE] = FileSize.ToString();
}
}
2)上传文件块指令
public class UploadData:ObjectMessage
{
public string FileMD5
{
get;
set;
}
public Beetle.ByteArraySegment Data
{
get;
set;
}
public override void FromProtocolData(HttpData httpbase)
{
FileMD5 = httpbase[CONSTVALUE.HEADER_MD5];
Data = httpbase.Content;
}
protected override void OnDisposed()
{
if (Data != null)
{
FileTransferPackage.BufferPool.Push(Data);
Data = null;
}
}
protected override void OnToProtocolData(HttpData httpbase)
{
httpbase.Command = CONSTVALUE.COMMAND_UPLOAD_DATA;
httpbase[CONSTVALUE.HEADER_MD5] = FileMD5;
httpbase.Content = Data;
}
}
3)返回值指令
public class Result :ObjectMessage
{
public string FileMD5
{
get;
set;
}
public bool Error
{
get;
set;
}
public string ErrorDetail
{
get;
set;
}
public override void FromProtocolData(HttpData httpbase)
{
ErrorDetail = httpbase[CONSTVALUE.HEADER_STATUS_DETAIL];
Error = httpbase[CONSTVALUE.HEADER_STATUS] == CONSTVALUE.VALUE_SUCCESS;
FileMD5 = httpbase[CONSTVALUE.HEADER_MD5];
}
protected override void OnDisposed()
{
}
protected override void OnToProtocolData(HttpData httpbase)
{
httpbase.Command = CONSTVALUE.COMMAND_RESULT;
if (Error)
{
httpbase[CONSTVALUE.HEADER_STATUS] = CONSTVALUE.VALUE_SUCCESS;
}
else
{
httpbase[CONSTVALUE.HEADER_STATUS] = CONSTVALUE.VALUE_ERROR;
}
httpbase[CONSTVALUE.HEADER_STATUS_DETAIL] = ErrorDetail;
httpbase[CONSTVALUE.HEADER_MD5] = FileMD5;
}
}
ObjectMessage是Beetle一个简化HTTP协议的扩展对象,它提供自定义Header和Body等功能.
文件读写器
既然需要处理文件块,那提供一些简单的文件块读取和写入方法是比较重要的.它不仅从设计解决功能的偶合度,还可以方便今后的利用.
1)UploadReader
public class UploadReader : IDisposable
{
public UploadReader(string file)
{
mStream = System.IO.File.OpenRead(file);
StringBuilder sb = new StringBuilder();
MD5 md5Hasher = MD5.Create();
foreach (Byte b in md5Hasher.ComputeHash(mStream))
sb.Append(b.ToString("x2").ToLower());
FileMD5 = sb.ToString();
mStream.Position = 0;
FileSize = mStream.Length;
FileName = System.IO.Path.GetFileName(file);
}
private System.IO.FileStream mStream = null;
public string FileName
{
get;
set;
}
public long LastReadLength
{
get;
set;
}
public long ReadLength
{
get;
set;
}
public long FileSize
{
get;
set;
}
public string FileMD5
{
get;
set;
}
public bool Completed
{
get
{
return mStream != null && ReadLength == mStream.Length;
}
}
public void Close()
{
if (mStream != null)
{
mStream.Close();
mStream.Dispose();
}
}
public void Reset()
{
mStream.Position = 0;
LastReadLength = 0;
ReadLength = 0;
}
public void Read(ByteArraySegment segment)
{
int loads = mStream.Read(segment.Array, 0, FileTransferPackage.BUFFER_SIZE);
segment.SetInfo(0, loads);
ReadLength += loads;
}
public void Dispose()
{
mStream.Dispose();
}
public override string ToString()
{
string value= string.Format("{0}(MD5:{4})\r\n\r\n[{1}/{2}({3}/秒)]",FileName,ReadLength,FileSize,ReadLength-LastReadLength,FileMD5);
if (!Completed)
{
LastReadLength = ReadLength;
}
return value;
}
}
UploadReader的功能主要是把文件流读取到指定大小的Buffer中,并提供方法获取当前的读取情况
2)UploadWriter
public class UploadWriter
{
public UploadWriter(string rootPath, string filename,string fileMD5,long size)
{
mFullName = rootPath + filename;
FileName = filename;
FileMD5 = fileMD5;
Size = size;
}
private string mFullName;
private System.IO.FileStream mStream;
public System.IO.FileStream Stream
{
get
{
if (mStream == null)
{
mStream = System.IO.File.Create(mFullName+".up");
}
return mStream;
}
}
public long WriteLength
{
get;
set;
}
public long LastWriteLength
{
get;
set;
}
public long Size
{
get;
set;
}
public string FileName
{
get;
set;
}
public string FileMD5
{
get;
set;
}
public bool Write(ByteArraySegment segment)
{
Stream.Write(segment.Array, 0, segment.Count);
WriteLength += segment.Count;
Stream.Flush();
if (WriteLength == Size)
{
Stream.Close();
if (System.IO.File.Exists(mFullName))
System.IO.File.Delete(mFullName);
System.IO.File.Move(mFullName + ".up", mFullName);
return true;
}
return false;
}
}
UploadWriter的功能主要是把文件写入到临时文件中,写入完成后再更改相应的名称,为了方便查询同样也提供了一些写入情况信息.
服务端代码
如果有了解过Beetle的服务端制定的话,那服务端的实现是非常简单的,只需要写一个对象承继ServerBase并实现数据接收方法处理即可以,接收的数据会会自动转换成之前定义的消息对象,而服务端内部处理的细节是完全不用关心.
protected override void OnMessageReceive(Beetle.PacketRecieveMessagerArgs e)
{
base.OnMessageReceive(e);
if (e.Message is Protocol.Upload)
{
OnUpload(e.Channel, e.Message as Protocol.Upload);
}
else if (e.Message is Protocol.UploadData)
{
OnUploadData(e.Channel, e.Message as Protocol.UploadData);
}
}
private Protocol.Result GetErrorResult(string detail)
{
Protocol.Result result = new Protocol.Result();
result.Error = true;
result.ErrorDetail = detail;
return result;
}
private void OnUpload(Beetle.TcpChannel channel, Protocol.Upload e)
{
Protocol.Result result;
if (mTask[e.FileMD5] != null)
{
result = GetErrorResult( "该文件正在上传任务中!");
channel.Send(result);
return;
}
UploadWriter writer = new UploadWriter(mRootPath, e.FileName, e.FileMD5, e.FileSize);
lock (mTask)
{
mTask[e.FileMD5] = writer;
}
result = new Protocol.Result();
channel.Send(result);
}
private void OnUploadData(Beetle.TcpChannel channel, Protocol.UploadData e)
{
using (e)
{
Protocol.Result result;
UploadWriter writer = (UploadWriter)mTask[e.FileMD5];
if (writer == null)
{
result = GetErrorResult("上传任务不存在!");
channel.Send(result);
return;
}
if (writer.Write(e.Data))
{
lock (mTask)
{
mTask.Remove(e.FileMD5);
}
}
result = new Protocol.Result();
result.FileMD5 = writer.FileMD5;
channel.Send(result);
}
}
当接收到客户求上传请求后会建立对应MD5的文件写入器,后面文件块的上传写入相关对象即可.
客户端代码
Beetle对于Client的支持也是非常简单方便,只需要定义一个TcpChannel直接发送定义的对象消息并获取服务器端返回的消息即可.
1 private void OnUpload(object state)
2 {
3 Lib.UploadReader reader = (Lib.UploadReader)state;
4 try
5 {
6 IsUpload = true;
7 Lib.Protocol.Upload upload = new Lib.Protocol.Upload();
8 upload.FileMD5 = reader.FileMD5;
9 upload.FileName = reader.FileName;
10 upload.FileSize = reader.FileSize;
11 Lib.Protocol.Result result = mClient.Send<Lib.Protocol.Result>(upload);
12 if (result.Error)
13 {
14 mLastError = result.ErrorDetail;
15 return;
16 }
17 while (!reader.Completed)
18 {
19 mLastError = "文件上传中...";
20 Lib.Protocol.UploadData data = new Lib.Protocol.UploadData();
21 data.Data = Lib.FileTransferPackage.BufferPool.Pop();
22 data.FileMD5 = reader.FileMD5;
23 reader.Read(data.Data);
24 result = mClient.Send<Lib.Protocol.Result>(data);
25 if (result.Error)
26 {
27 mLastError = result.ErrorDetail;
28 return;
29 }
30 }
31 mLastError = "文件上传完成!";
32
33 }
34 catch (Exception e_)
35 {
36 mLastError = e_.Message;
37 }
38 mReader.Reset();
39 IsUpload = false;
40
41 }
整个过程只需要一个方法却可完成,首先把需要上传的文件信息发送到服务器,当服务器确认后不停地把文件块信息输送到服务端即可.
使用测试
下载代码
版权声明：本文为yuluoxingkong原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。