asp.net抓取其他站点的部分数据,然后存储到数据库,要有一个成功的案例,越详细越好,谢谢!
比如我想抓取某一个网页的部分数据,然后存储起来,就是小偷程序。
比如我想抓取某一个网页的部分数据,然后存储起来,就是小偷程序。
答案:很长了。不是一两句可以写完。给你下面的代码已经带模拟登陆抓取功能了。我现在的网站都是用这个。#region GetHtml
/// <summary>
/// 网站Cookies
/// </summary>
private string _cookieHeader = string.Empty;
public string CookieHeader
{
get
{
return _cookieHeader;
}
set
{
_cookieHeader = value;
}
}
/// <summary>
/// 网站编码
/// </summary>
private string _code = string.Empty;
public string Code
{
get { return _code; }
set { _code = value; }
}
private string _pageContent = string.Empty;
public string PageContent
{
get { return _pageContent; }
set { _pageContent = value; }
}
private Dictionary<string, string> _para = new Dictionary<string, string>();
public Dictionary<string, string> Para
{
get { return _para; }
set { _para = value; }
}/**/
/// <summary>
/// 功能描述:模拟登录页面,提交登录数据进行登录,并记录Header中的cookie
/// </summary>
/// <param >登录数据提交的页面地址</param>
/// <param >用户登录数据</param>
/// <param >引用地址</param>
/// <param >网站编码</param>
/// <returns>可以返回页面内容或不返回</returns>
public string PostData(string strURL, string strArgs, string strReferer, string code, string method)
{
return PostData(strURL, strArgs, strReferer, code, method, string.Empty);
}
public string PostData(string strURL, string strArgs, string strReferer, string code, string method, string contentType)
{
try
{
string strResult = "";
HttpWebRequest myHttpWebRequest = (HttpWebRequest)WebRequest.Create(strURL);
myHttpWebRequest.AllowAutoRedirect = true;
myHttpWebRequest.KeepAlive = true;
myHttpWebRequest.Accept = "image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, application/vnd.ms-excel, application/msword, application/x-shockwave-flash, */*";
myHttpWebRequest.Referer = strReferer;
myHttpWebRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; Maxthon; .NET CLR 2.0.50727)";if (string.IsNullOrEmpty(contentType))
{
myHttpWebRequest.ContentType = "application/x-www-form-urlencoded";
}
else
{
myHttpWebRequest.ContentType = "contentType";
}myHttpWebRequest.Method = method;
myHttpWebRequest.Headers.Add("Accept-Encoding", "gzip, deflate");
if (myHttpWebRequest.CookieContainer == null)
{
myHttpWebRequest.CookieContainer = new CookieContainer();
//strResult = "wo funccccccc";
}if (this.CookieHeader.Length > 0)
{
myHttpWebRequest.Headers.Add("cookie:" + this.CookieHeader);
myHttpWebRequest.CookieContainer.SetCookies(new Uri(strURL), this.CookieHeader);
//return strResult += this.CookieHeader.ToString();
}byte[] postData = Encoding.GetEncoding(code).GetBytes(strArgs);
myHttpWebRequest.ContentLength = postData.Length;System.IO.Stream PostStream = myHttpWebRequest.GetRequestStream();
PostStream.Write(postData, 0, postData.Length);
PostStream.Close();HttpWebResponse response = null;
System.IO.StreamReader sr = null;
response = (HttpWebResponse)myHttpWebRequest.GetResponse();if (myHttpWebRequest.CookieContainer != null)
{
this.CookieHeader = myHttpWebRequest.CookieContainer.GetCookieHeader(new Uri(strURL));
}sr = new System.IO.StreamReader(response.GetResponseStream(), Encoding.GetEncoding(code));?//?//utf-8
&n