当前位置:编程学习 > C#/ASP.NET >>

asp.net抓取其他站点的部分数据,然后存储到数据库,要有一个成功的案例,越详细越好,谢谢!

比如我想抓取某一个网页的部分数据,然后存储起来,就是小偷程序。
答案:很长了。不是一两句可以写完。给你下面的代码已经带模拟登陆抓取功能了。我现在的网站都是用这个。

 #region GetHtml
    /// <summary>
    /// 网站Cookies
    /// </summary>
    private string _cookieHeader = string.Empty;
    public string CookieHeader
    {
        get
        {
            return _cookieHeader;
        }
        set
        {
            _cookieHeader = value;
        }
    }
    /// <summary>
    /// 网站编码
    /// </summary>
    private string _code = string.Empty;
    public string Code
    {
        get { return _code; }
        set { _code = value; }
    }
    private string _pageContent = string.Empty;
    public string PageContent
    {
        get { return _pageContent; }
        set { _pageContent = value; }
    }
    private Dictionary<string, string> _para = new Dictionary<string, string>();
    public Dictionary<string, string> Para
    {
        get { return _para; }
        set { _para = value; }
    }

    /**/
    /// <summary>
    /// 功能描述:模拟登录页面,提交登录数据进行登录,并记录Header中的cookie
    /// </summary>
    /// <param >登录数据提交的页面地址</param>
    /// <param >用户登录数据</param>
    /// <param >引用地址</param>
    /// <param >网站编码</param>
    /// <returns>可以返回页面内容或不返回</returns>
    public string PostData(string strURL, string strArgs, string strReferer, string code, string method)
    {
        return PostData(strURL, strArgs, strReferer, code, method, string.Empty);
    }
    public string PostData(string strURL, string strArgs, string strReferer, string code, string method, string contentType)
    {
        try
        {
            string strResult = "";
            HttpWebRequest myHttpWebRequest = (HttpWebRequest)WebRequest.Create(strURL);
            myHttpWebRequest.AllowAutoRedirect = true;
            myHttpWebRequest.KeepAlive = true;
            myHttpWebRequest.Accept = "image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, application/vnd.ms-excel, application/msword, application/x-shockwave-flash, */*";
            myHttpWebRequest.Referer = strReferer;


            myHttpWebRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; Maxthon; .NET CLR 2.0.50727)";

            if (string.IsNullOrEmpty(contentType))
            {
                myHttpWebRequest.ContentType = "application/x-www-form-urlencoded";
            }
            else
            {
                myHttpWebRequest.ContentType = "contentType";
            }

            myHttpWebRequest.Method = method;

            myHttpWebRequest.Headers.Add("Accept-Encoding", "gzip, deflate");

            if (myHttpWebRequest.CookieContainer == null)
            {
                myHttpWebRequest.CookieContainer = new CookieContainer();
                //strResult = "wo funccccccc";
            }

            if (this.CookieHeader.Length > 0)
            {
                myHttpWebRequest.Headers.Add("cookie:" + this.CookieHeader);
                myHttpWebRequest.CookieContainer.SetCookies(new Uri(strURL), this.CookieHeader);
                //return strResult += this.CookieHeader.ToString();
            }

            byte[] postData = Encoding.GetEncoding(code).GetBytes(strArgs);
            myHttpWebRequest.ContentLength = postData.Length;

            System.IO.Stream PostStream = myHttpWebRequest.GetRequestStream();
            PostStream.Write(postData, 0, postData.Length);
            PostStream.Close();

            HttpWebResponse response = null;
            System.IO.StreamReader sr = null;
            response = (HttpWebResponse)myHttpWebRequest.GetResponse();

            if (myHttpWebRequest.CookieContainer != null)
            {
                this.CookieHeader = myHttpWebRequest.CookieContainer.GetCookieHeader(new Uri(strURL));
            }

            sr = new System.IO.StreamReader(response.GetResponseStream(), Encoding.GetEncoding(code));?//?//utf-8
          &n

上一个:php 抽奖程序
下一个:设计一个程序

CopyRight © 2012 站长网 编程知识问答 www.zzzyk.com All Rights Reserved
部份技术文章来自网络,