公司编辑妹子需要爬取网页内容，叫我帮忙做了一简单的爬取工具
这是爬取网页内容，像是这对大家来说都是不难得，但是在这里有一些小改动，代码献上，大家参考
private string GetHttpWebRequest(string url)
{
HttpWebResponse result;
string strHTML = string.Empty;
try
{
Uri uri = new Uri(url);
WebRequest webReq = WebRequest.Create(uri);
WebResponse webRes = webReq.GetResponse();
HttpWebRequest myReq = (HttpWebRequest)webReq;
myReq.UserAgent = "User-Agent:Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705";
myReq.Accept = "*/*";
myReq.KeepAlive = true;
myReq.Headers.Add("Accept-Language","zh-cn,en-us;q=0.5");
result = (HttpWebResponse)myReq.GetResponse();
Stream receviceStream = result.GetResponseStream();
StreamReader readerOfStream = new StreamReader(receviceStream,System.Text.Encoding.GetEncoding("utf-8"));
strHTML = readerOfStream.ReadToEnd();
readerOfStream.Close();
receviceStream.Close();
result.Close();
}
catch
{
Uri uri = new Uri(url);
WebRequest webReq = WebRequest.Create(uri);
HttpWebRequest myReq = (HttpWebRequest)webReq;
myReq.UserAgent = "User-Agent:Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705";
myReq.Accept = "*/*";
myReq.KeepAlive = true;
myReq.Headers.Add("Accept-Language",en-us;q=0.5");
//result = (HttpWebResponse)myReq.GetResponse();
try
{
result = (HttpWebResponse)myReq.GetResponse();
}
catch (WebException ex)
{
result = (HttpWebResponse)ex.Response;
}
Stream receviceStream = result.GetResponseStream();
StreamReader readerOfStream = new StreamReader(receviceStream,System.Text.Encoding.GetEncoding("gb2312"));
strHTML = readerOfStream.ReadToEnd();
readerOfStream.Close();
receviceStream.Close();
result.Close();
}
return strHTML;
}
这是根据url爬取网页远吗，有一些小改动，很多网页有不同的编码格式，甚至有些网站做了反爬取的防范，这个方法经过能够改动也能爬去
以下是爬取网页所有的网址链接
/// <summary>
/// 提取HTML代码中的网址
/// </summary>
/// <param name="htmlCode"></param>
/// <returns></returns>
private static List<string> GetHyperLinks(string htmlCode,string url)
{
ArrayList al = new ArrayList();
bool IsGenxin = false;
StringBuilder weburlSB = new StringBuilder();//sql
StringBuilder linkSb = new StringBuilder();//展示数据
List<string> Weburllistzx = new List<string>();//新增
List<string> Weburllist = new List<string>();//旧的
string ProductionContent = htmlCode;
Regex reg = new Regex(@"http(s)?://([\w-]+\.)+[\w-]+/?");
string wangzhanyuming = reg.Match(url,0).Value;
MatchCollection mc = Regex.Matches(ProductionContent.Replace("href=\"/","href=\"" + wangzhanyuming).Replace("href='/","href='" + wangzhanyuming).Replace("href=/","href=" + wangzhanyuming).Replace("href=\"./","href=\"" + wangzhanyuming),@"<[aA][^>]* href=[^>]*>",RegexOptions.Singleline);
int Index = 1;
foreach (Match m in mc)
{
MatchCollection mc1 = Regex.Matches(m.Value,@"[a-zA-z]+://[^\s]*",RegexOptions.Singleline);
if (mc1.Count > 0)
{
foreach (Match m1 in mc1)
{
string linkurlstr = string.Empty;
linkurlstr = m1.Value.Replace("\"","").Replace("'","").Replace(">","").Replace(";","");
weburlSB.Append("$-$");
weburlSB.Append(linkurlstr);
weburlSB.Append("$_$");
if (!Weburllist.Contains(linkurlstr) && !Weburllistzx.Contains(linkurlstr))
{
IsGenxin = true;
Weburllistzx.Add(linkurlstr);
linkSb.AppendFormat("{0}<br/>",linkurlstr);
}
}
}
else
{
if (m.Value.IndexOf("javascript") == -1)
{
string amstr = string.Empty;
string wangzhanxiangduilujin = string.Empty;
wangzhanxiangduilujin = url.Substring(0,url.LastIndexOf("/") + 1);
amstr = m.Value.Replace("href=\"","href=\"" + wangzhanxiangduilujin).Replace("href='","href='" + wangzhanxiangduilujin);
MatchCollection mc11 = Regex.Matches(amstr,RegexOptions.Singleline);
foreach (Match m1 in mc11)
{
string linkurlstr = string.Empty;
linkurlstr = m1.Value.Replace("\"","");
weburlSB.Append("$-$");
weburlSB.Append(linkurlstr);
weburlSB.Append("$_$");
if (!Weburllist.Contains(linkurlstr) && !Weburllistzx.Contains(linkurlstr))
{
IsGenxin = true;
Weburllistzx.Add(linkurlstr);
linkSb.AppendFormat("{0}<br/>",linkurlstr);
}
}
}
}
Index++;
}
return Weburllistzx;
}
这块的技术其实就是简单的使用了正则去匹配！接下来献上获取标题，以及存储到xml文件的方法
/// <summary>
/// // 把网址写入xml文件
/// </summary>
/// <param name="strURL"></param>
/// <param name="alHyperLinks"></param>
private static void WritetoXml(string strURL,List<string> alHyperLinks)
{
XmlTextWriter writer = new XmlTextWriter(@"D:\HyperLinks.xml",Encoding.UTF8);
writer.Formatting = Formatting.Indented;
writer.WriteStartDocument(false);
writer.WriteDocType("HyperLinks",null,"urls.dtd",null);
writer.WriteComment("提取自" + strURL + "的超链接");
writer.WriteStartElement("HyperLinks");
writer.WriteStartElement("HyperLinks",null);
writer.WriteAttributeString("DateTime",DateTime.Now.ToString());
foreach (string str in alHyperLinks)
{
string title = GetDomain(str);
string body = str;
writer.WriteElementString(title,body);
}
writer.WriteEndElement();
writer.WriteEndElement();
writer.Flush();
writer.Close();
}
/// <summary>
/// 获取网址的域名后缀
/// </summary>
/// <param name="strURL"></param>
/// <returns></returns>
private static string GetDomain(string strURL)
{
string retVal;
string strRegex = @"(\.com/|\.net/|\.cn/|\.org/|\.gov/)";
Regex r = new Regex(strRegex,RegexOptions.IgnoreCase);
Match m = r.Match(strURL);
retVal = m.ToString();
strRegex = @"\.|/$";
retVal = Regex.Replace(retVal,strRegex,"").ToString();
if (retVal == "")
retVal = "other";
return retVal;
}
/// <summary>
/// 获取标题
/// </summary>
/// <param name="html"></param>
/// <returns></returns>
private static string GetTitle(string html)
{
string titleFilter = @"<title>[\s\S]*?</title>";
string h1Filter = @"<h1.*?>.*?</h1>";
string clearFilter = @"<.*?>";
string title = "";
Match match = Regex.Match(html,titleFilter,RegexOptions.IgnoreCase);
if (match.Success)
{
title = Regex.Replace(match.Groups[0].Value,clearFilter,"");
}
// 正文的标题一般在h1中，比title中的标题更干净
match = Regex.Match(html,h1Filter,RegexOptions.IgnoreCase);
if (match.Success)
{
string h1 = Regex.Replace(match.Groups[0].Value,"");
if (!String.IsNullOrEmpty(h1) && title.StartsWith(h1))
{
title = h1;
}
}
return title;
}
这就是所用的全部方法，还是有很多需要改进之处！大家如果有发现不足之处还请指出，谢谢！
以上就是本文的全部内容，希望对大家的学习有所帮助，也希望大家多多支持编程小技巧。
版权声明：本文内容由互联网用户自发贡献，该文观点与技术仅代表作者本人。本站仅提供信息存储空间服务，不拥有所有权，不承担相关法律责任。如发现本站有涉嫌侵权/违法违规的内容， 请发送邮件至 dio@foxmail.com 举报，一经查实，本站将立刻删除。
